diff --git a/.github/workflows/perf-ab.yaml b/.github/workflows/perf-ab.yaml new file mode 100644 index 000000000..487ea0610 --- /dev/null +++ b/.github/workflows/perf-ab.yaml @@ -0,0 +1,83 @@ +name: perf-ab + +# Tier B of the storage measurement harness (hack/perf-ab.sh): a paired A/B of +# the branch against its merge-base, rebuilt in the same run on the same +# runner. GitHub runners are noisy, so INCONCLUSIVE is expected often here and +# is reported, not hidden; a local run on a quiet machine is the authoritative +# one. Only a REGRESSION verdict fails the check. + +on: + workflow_dispatch: + inputs: + base: + description: "Commit to compare against (default: merge-base with origin/main)" + required: false + type: string + pairs: + description: "Interleaved (base, head) rounds" + required: false + type: string + default: "10" + schedule: + - cron: "17 3 * * *" + pull_request: + types: [opened, synchronize, reopened, labeled] + +jobs: + perf-ab: + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'perf') + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - name: Resolve base + id: base + run: | + if [ -n "${{ inputs.base }}" ]; then + echo "sha=${{ inputs.base }}" >> "$GITHUB_OUTPUT" + elif [ "${{ github.event_name }}" = "schedule" ]; then + echo "sha=$(git rev-list -n 1 --before='1 day ago' origin/main)" >> "$GITHUB_OUTPUT" + else + echo "sha=$(git merge-base origin/main HEAD)" >> "$GITHUB_OUTPUT" + fi + - name: Paired A/B + id: ab + env: + BASE: ${{ steps.base.outputs.sha }} + PAIRS: ${{ inputs.pairs || '10' }} + PERF_AB_OUT_DIR: ${{ runner.temp }}/perf-ab + PERF_AB_ALLOW_NOISY: "1" + run: | + set +e + make perf-ab + echo "rc=$?" >> "$GITHUB_OUTPUT" + - name: Report verdict + if: always() + env: + OUT: ${{ runner.temp }}/perf-ab + RC: ${{ steps.ab.outputs.rc }} + run: | + { + echo "## perf-ab: $(cat "$OUT/verdict.txt" 2>/dev/null || echo 'no verdict (driver failed)')" + echo + echo "base \`${{ steps.base.outputs.sha }}\` vs head \`${{ github.sha }}\`, exit $RC (0 pass, 1 regression, 2 config mismatch, 3 inconclusive, 4 underpowered)" + echo + if [ -f "$OUT/verdict-table.txt" ]; then echo '```'; cat "$OUT/verdict-table.txt"; echo '```'; fi + if [ -f "$OUT/schedule.txt" ]; then echo '
schedule'; echo; echo '```'; cat "$OUT/schedule.txt"; echo '```'; echo '
'; fi + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload rounds + if: always() + uses: actions/upload-artifact@v4 + with: + name: perf-ab-${{ github.run_id }} + path: ${{ runner.temp }}/perf-ab + if-no-files-found: ignore + - name: Fail on regression + if: steps.ab.outputs.rc == '1' + run: exit 1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4d9781f3f..1a9a5c065 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,3 +3,16 @@ The Kubescape project manages this document in the central project repository. Go to the [centralized CONTRIBUTING.md](https://github.com/kubescape/project-governance/blob/main/CONTRIBUTING.md) + +## Storage hot-path changes + +A PR touching `pkg/registry/file/{storage,singlewriter,containerprofile_*,sqlite}.go` +must carry two numbers (see `docs/features/storage-measurement-harness.md`): + +- **Work budgets (Tier A).** `go test ./...` includes `TestWorkBudget`. If the + golden `pkg/registry/file/testdata/workbudget.golden.json` changes, the + commit message states each row's delta and why; the reviewer checks the + attribution. +- **Paired A/B (Tier B).** Run `make perf-ab` on a quiet machine and quote its + verdict line verbatim in the PR description. `INCONCLUSIVE` and + `UNDERPOWERED` are not `PASS`. diff --git a/Makefile b/Makefile index 9937eeebb..31901cbe0 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ TAG?=test IMAGE?=quay.io/kubescape/$(BINARY_NAME) -.PHONY: build test docker-build docker-push +.PHONY: build test perf-ab docker-build docker-push build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o $(BINARY_NAME) @@ -13,6 +13,13 @@ build: test: go test ./... +# Paired A/B of HEAD against its merge-base on this machine (Tier B of the +# storage measurement harness). Mandatory before SHIP for any change under +# pkg/registry/file/{storage,singlewriter,containerprofile_*,sqlite}.go; quote +# its verdict line in the PR. BASE= and PAIRS= override the defaults. +perf-ab: + hack/perf-ab.sh + docker-build: docker buildx build --platform linux/amd64 -t $(IMAGE):$(TAG) --load -f $(DOCKERFILE_PATH) . docker-push: diff --git a/build/Dockerfile b/build/Dockerfile index e57051966..673e9fa0f 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -9,6 +9,11 @@ RUN --mount=target=. \ --mount=type=cache,target=/go/pkg \ GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /out/migration ./cmd/migration/main.go +RUN --mount=target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg \ + GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /out/cpexport ./cmd/cpexport/main.go + RUN --mount=target=. \ --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg \ @@ -18,6 +23,7 @@ FROM gcr.io/distroless/static-debian13:nonroot COPY --from=builder /out/storage /usr/bin/storage COPY --from=builder /out/migration /usr/bin/migration +COPY --from=builder /out/cpexport /usr/bin/cpexport ARG image_version ENV RELEASE=$image_version diff --git a/cmd/cpexport/main.go b/cmd/cpexport/main.go new file mode 100644 index 000000000..4444a7ae5 --- /dev/null +++ b/cmd/cpexport/main.go @@ -0,0 +1,65 @@ +package main + +// cpexport writes every ContainerProfile the SQLite-native backend +// (config.ContainerProfileSqliteBackend) holds in the payloads table back +// as the legacy gob payload file an older storage binary expects. It is the +// first half of the rollback order of +// .omc/plans/full-acid-storage-architecture.md §8.4 — export, THEN +// downgrade — because an older binary's get() deletes the metadata row of +// any ContainerProfile whose file is missing, destroying the object. +// +// Run it with the storage server stopped (scale the deployment to 0, then +// run it against the PVC), never alongside a serving process: a write that +// lands after a key was exported leaves the old binary a stale file. +// +// cpexport [-db /data/metadata.sq3] [-root /data] [-dry-run] + +import ( + "context" + "flag" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition/install" + "github.com/kubescape/storage/pkg/registry/file" + "github.com/spf13/afero" + "k8s.io/apimachinery/pkg/runtime" +) + +func main() { + root := flag.String("root", file.DefaultStorageRoot, "storage root directory holding the payload files") + db := flag.String("db", "", "SQLite database path (default /metadata.sq3)") + dryRun := flag.Bool("dry-run", false, "decode and count without writing any file") + batch := flag.Int("batch", file.DefaultMigrationBatchSize, "rows read per query") + flag.Parse() + if *db == "" { + *db = filepath.Join(*root, "metadata.sq3") + } + // The pool retries an unopenable database every 5 s until ctx expires + // (sqlitemigration.Pool.Take); a missing database is always a wrong + // -db/-root, not something to wait an hour for. + if _, err := os.Stat(*db); err != nil { + fmt.Fprintf(os.Stderr, "cpexport: database: %v\n", err) + os.Exit(1) + } + + sch := runtime.NewScheme() + install.Install(sch) + pool := file.NewPoolWithOptions(*db, file.PoolOptions{Size: 2, BusyTimeout: file.DefaultBusyTimeout}) + defer func() { _ = pool.Close() }() + + ctx, cancel := context.WithTimeout(context.Background(), time.Hour) + defer cancel() + report, err := file.ExportContainerProfiles(ctx, pool, afero.NewOsFs(), *root, sch, file.ContainerProfileExportOptions{BatchSize: *batch, DryRun: *dryRun}) + if err != nil { + fmt.Fprintf(os.Stderr, "cpexport: %v\n", err) + os.Exit(1) + } + fmt.Printf("cpexport: exported=%d legacySkipped=%d undecodable=%d dryRun=%v elapsed=%s\n", + report.Exported, report.LegacySkipped, report.Undecodable, report.DryRun, report.Elapsed) + if report.Undecodable > 0 { + os.Exit(2) + } +} diff --git a/cmd/cpexport/main_test.go b/cmd/cpexport/main_test.go new file mode 100644 index 000000000..d6ec66242 --- /dev/null +++ b/cmd/cpexport/main_test.go @@ -0,0 +1,254 @@ +package main + +// cpexport is driven as the operator would drive it: the real built binary, +// a real root directory and database, exit codes and the report line. The +// export's semantics per row shape are pinned in +// pkg/registry/file/sqliteobject_export_test.go; the scale run is +// TestExport_Scale_BinaryAgainstMigratedCorpus there. + +import ( + "bytes" + "context" + "encoding/gob" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "sync" + "testing" + "time" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/kubescape/storage/pkg/apis/softwarecomposition/install" + "github.com/kubescape/storage/pkg/registry/file" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/uuid" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitex" +) + +var ( + binOnce sync.Once + binPath string + binErr error +) + +func buildBinary(t *testing.T) string { + t.Helper() + binOnce.Do(func() { + dir, err := os.MkdirTemp("", "cpexport-bin") + if err != nil { + binErr = err + return + } + bin := filepath.Join(dir, "cpexport") + cmd := exec.Command("go", "build", "-o", bin, ".") + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + binErr = fmt.Errorf("%w: %s", err, stderr.String()) + return + } + binPath = bin + }) + require.NoError(t, binErr, "go build cmd/cpexport") + return binPath +} + +var reportRE = regexp.MustCompile(`cpexport: exported=(\d+) legacySkipped=(\d+) undecodable=(\d+) dryRun=(true|false) elapsed=`) + +type result struct { + code int + exported, legacySkipped, undecodable int + dryRun bool + stdout, stderr string +} + +// run drives the binary with a bound: the export of a few rows is +// sub-second, so a run that takes longer is a hang (the pool retrying an +// unopenable database), not a slow machine. +func run(t *testing.T, bin string, args ...string) result { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, bin, args...) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + require.NoError(t, ctx.Err(), "cpexport did not exit within the bound (hung): args=%v", args) + r := result{stdout: stdout.String(), stderr: stderr.String()} + var exitErr *exec.ExitError + switch { + case err == nil: + case errors.As(err, &exitErr): + r.code = exitErr.ExitCode() + default: + t.Fatalf("run cpexport: %v", err) + } + if m := reportRE.FindStringSubmatch(r.stdout); m != nil { + r.exported, _ = strconv.Atoi(m[1]) + r.legacySkipped, _ = strconv.Atoi(m[2]) + r.undecodable, _ = strconv.Atoi(m[3]) + r.dryRun = m[4] == "true" + } + return r +} + +const prefix = "/spdx.softwarecomposition.kubescape.io/containerprofile/" + +// migratedRoot seeds n ContainerProfiles through the legacy StorageImpl over +// a real root, migrates them into the ObjectStore schema, and returns the +// root, the database path and the objects by name — with every legacy file +// removed, so the export has something to do. +func migratedRoot(t *testing.T, n int) (root, dbPath string, objs map[string]*softwarecomposition.ContainerProfile) { + t.Helper() + root = filepath.Join(t.TempDir(), "data") + require.NoError(t, os.MkdirAll(root, 0755)) + dbPath = filepath.Join(root, "metadata.sq3") + sch := runtime.NewScheme() + install.Install(sch) + pool := file.NewPoolWithOptions(dbPath, file.PoolOptions{Size: 4, BusyTimeout: 5 * time.Second}) + fs := afero.NewOsFs() + legacy := file.NewStorageImpl(fs, root, pool, file.NewWatchDispatcher(), sch) + ctx := context.Background() + objs = map[string]*softwarecomposition.ContainerProfile{} + for i := 0; i < n; i++ { + name := fmt.Sprintf("profile-%03d", i) + p := &softwarecomposition.ContainerProfile{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "ns", UID: uuid.NewUUID(), + Labels: map[string]string{"n": strconv.Itoa(i)}}, + Spec: softwarecomposition.ContainerProfileSpec{Execs: []softwarecomposition.ExecCalls{{Path: "/bin/" + name}}}, + } + out := &softwarecomposition.ContainerProfile{} + require.NoError(t, legacy.Create(ctx, prefix+"ns/"+name, p, out, 0)) + objs[name] = out + } + gate, err := file.NewWriteGate(ctx, pool) + require.NoError(t, err) + report, err := file.MigrateContainerProfiles(ctx, pool, gate, fs, root, sch, file.ContainerProfileMigrationOptions{}) + require.NoError(t, err) + require.Equal(t, n, report.Count(file.MigrationShapeMigrated), "%v", report.Counts) + require.NoError(t, gate.Close()) + require.NoError(t, pool.Close()) + for name := range objs { + require.NoError(t, os.Remove(payloadPath(root, name))) + } + return root, dbPath, objs +} + +func payloadPath(root, name string) string { + return filepath.Join(root, prefix, "ns", name) + file.GobExt +} + +func decodeFile(t *testing.T, path string) *softwarecomposition.ContainerProfile { + t.Helper() + b, err := os.ReadFile(path) + require.NoError(t, err) + obj := &softwarecomposition.ContainerProfile{} + require.NoError(t, gob.NewDecoder(bytes.NewReader(b)).Decode(obj)) + return obj +} + +func payloadFiles(t *testing.T, root string) int { + t.Helper() + n := 0 + require.NoError(t, filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err == nil && !info.IsDir() && file.IsPayloadFile(path) { + n++ + } + return err + })) + return n +} + +func TestCpexport_DryRunThenReal(t *testing.T) { + bin := buildBinary(t) + root, dbPath, objs := migratedRoot(t, 30) + require.Equal(t, 0, payloadFiles(t, root)) + + dry := run(t, bin, "-root", root, "-db", dbPath, "-dry-run") + require.Equal(t, 0, dry.code, "stderr: %s", dry.stderr) + require.True(t, dry.dryRun, dry.stdout) + require.Equal(t, 30, dry.exported, dry.stdout) + require.Equal(t, 0, dry.legacySkipped) + require.Equal(t, 0, dry.undecodable) + require.Equal(t, 0, payloadFiles(t, root), "dry-run wrote nothing") + + // The default -db is /metadata.sq3. + real := run(t, bin, "-root", root) + require.Equal(t, 0, real.code, "stderr: %s", real.stderr) + require.False(t, real.dryRun) + require.Equal(t, 30, real.exported, real.stdout) + require.Equal(t, 30, payloadFiles(t, root)) + for name, want := range objs { + got := decodeFile(t, payloadPath(root, name)) + require.Equal(t, want.Name, got.Name) + require.Equal(t, want.UID, got.UID) + require.Equal(t, want.ResourceVersion, got.ResourceVersion) + require.Equal(t, want.Labels, got.Labels) + require.Equal(t, want.Spec.Execs, got.Spec.Execs) + _, err := os.Stat(payloadPath(root, name) + ".t") + require.True(t, os.IsNotExist(err), "no staging file left for %s", name) + } + + // Idempotent: a second export rewrites the same files. + again := run(t, bin, "-root", root, "-db", dbPath, "-batch", "7") + require.Equal(t, 0, again.code) + require.Equal(t, 30, again.exported) + require.Equal(t, 30, payloadFiles(t, root)) +} + +// An undecodable payload is skipped and reported, every other row is still +// exported, and the exit code is 2 so an operator's script notices. +func TestCpexport_UndecodableRowExits2(t *testing.T) { + bin := buildBinary(t) + root, dbPath, objs := migratedRoot(t, 5) + conn, err := sqlite.OpenConn(dbPath, sqlite.OpenReadWrite) + require.NoError(t, err) + require.NoError(t, sqlitex.Execute(conn, + `UPDATE payloads SET body = ? WHERE kind = ? AND namespace = ? AND name = ?`, + &sqlitex.ExecOptions{Args: []any{[]byte("{not json"), file.ContainerProfileKind, "ns", "profile-002"}})) + require.Equal(t, 1, conn.Changes()) + require.NoError(t, conn.Close()) + + r := run(t, bin, "-root", root, "-db", dbPath) + require.Equal(t, 2, r.code, "stdout: %s stderr: %s", r.stdout, r.stderr) + require.Equal(t, 4, r.exported, r.stdout) + require.Equal(t, 1, r.undecodable, r.stdout) + require.Equal(t, 4, payloadFiles(t, root)) + _, err = os.Stat(payloadPath(root, "profile-002")) + require.True(t, os.IsNotExist(err), "the undecodable row gets no file") + for _, name := range []string{"profile-000", "profile-001", "profile-003", "profile-004"} { + require.Equal(t, objs[name].UID, decodeFile(t, payloadPath(root, name)).UID) + } + // Dry-run reports the same and exits 2 as well. + dry := run(t, bin, "-root", root, "-db", dbPath, "-dry-run") + require.Equal(t, 2, dry.code) + require.Equal(t, 1, dry.undecodable) +} + +// A missing database (a wrong -db or -root) is exit 1 with the error on +// stderr and no report line — promptly. Without the up-front check the pool +// retries the open every 5 s until the binary's one-hour context expires. +func TestCpexport_MissingDatabaseExits1(t *testing.T) { + bin := buildBinary(t) + root := t.TempDir() + for _, db := range []string{filepath.Join(root, "missing-dir", "metadata.sq3"), filepath.Join(root, "metadata.sq3")} { + start := time.Now() + r := run(t, bin, "-root", root, "-db", db) + require.Equal(t, 1, r.code, "stdout: %s stderr: %s", r.stdout, r.stderr) + require.Contains(t, r.stderr, "cpexport: database:") + require.Empty(t, r.stdout) + require.Less(t, time.Since(start), 5*time.Second, "must fail before the pool's first retry") + } + // The default -db (/metadata.sq3) with an empty root: same. + r := run(t, bin, "-root", root) + require.Equal(t, 1, r.code) + require.Contains(t, r.stderr, "cpexport: database:") +} diff --git a/docs/features/containerprofile-sqlite-backend.md b/docs/features/containerprofile-sqlite-backend.md new file mode 100644 index 000000000..f15944fad --- /dev/null +++ b/docs/features/containerprofile-sqlite-backend.md @@ -0,0 +1,214 @@ +# ContainerProfile SQLite-native backend + +## Summary + +`config.Config.ContainerProfileSqliteBackend` (default `false`) selects a second storage backend +for the `containerprofiles` resource: `file.ObjectStore` +(`pkg/registry/file/sqliteobject_*.go`), which keeps the object's payload **inside the SQLite +database** next to its metadata row and writes metadata row, payload and `time_series` row in +**one transaction**. The legacy `StorageImpl` (metadata row in SQLite + gob payload file, two +commit points) stays the default and serves the other 13 kinds unchanged. + +This is the production form of `.omc/plans/full-acid-storage-architecture.md` (Revision 2): the +prototype's store, gate and checkpointer, plus the pieces the prototype left out — the startup +data migration (§8), the cleanup arm on rows (K-4), `GeneratedNetworkPolicyStorage` on the CP +store (§5.6 row 9) and the shared write gate for the 13 legacy kinds +(`docs/features/write-gate-sharing.md`) and the `cpexport` rollback tool. Still to come before the +flip: the legacy-file deletion step (§8.4) and the soak. + +## Why it matters + +Every member of the row/payload divergence family the storage investigation catalogued (E3, B16, +B19, B3, the PC-DIV shapes, `get()`'s self-repair deletes, the orphan temp-file reaper, the `Stat` +pre-check) exists because the metadata row and the payload file commit separately. With both in +one SQLite transaction none of those states is representable, and the compare-and-swap becomes one +`UPDATE … WHERE rv=:rv AND uid=:uid`. + +## What changes when the flag is on + +| Piece | Behaviour | +|---|---| +| Schema | Migrations 3–5 (always applied, additive): `metadata.rv INTEGER`, `metadata.uid TEXT` (both nullable; legacy kinds leave them NULL), `payloads(kind, namespace, name, encoding, body BLOB)` and `migration_state(name, state, counts, updated_at)` (the data migration's done-flag). | +| Payload | The object converted to `v1beta1` and JSON-marshalled (`payloads.encoding = json/v1beta1`); the original `TypeMeta` is preserved. | +| Create | Prepare (PreSave, checksum, encode) outside any transaction, then under the gate: in-transaction TS admission (`json_extract` on the base row; Completed/Full or TooLarge base → refused, transaction rolled back), `INSERT … ON CONFLICT DO NOTHING` (→ `KeyExists`), payload insert, `time_series` insert. | +| Update | `UPDATE metadata … WHERE rv=:rv AND uid=:uid`; `changes()==0` → conflict → backoff, re-read, retry (the single writer's loop). `changes()!=1` on the payloads `UPDATE` → `ROLLBACK` + `InternalError` (K-6). | +| Delete | metadata `RETURNING` + payloads + `time_series` in one transaction; absent key → `KeyNotFound`. | +| Get / List | One `SELECT … JOIN payloads` (autocommit, WAL snapshot, no per-key lock). Metadata-only GET/LIST run the legacy statements, so continue tokens stay rowids. | +| Consolidation | `WithConnection` hands the pass a read handle; `BeginTransaction` opens a staged write set; base save (CAS), `time_series` rewrite and the processed-TS deletes (each with its own `rv`/`uid` predicate, R4) commit together under the gate; any failed CAS rolls the whole tick back and the pass retries once. | +| Write gate | Caller-side two-lane FIFO ticket semaphore with `highBurstLimit` fairness, ctx-cancellable, owning one dedicated pool connection; prepare → ticket → `BEGIN IMMEDIATE` → `COMMIT` → release → dispatch. Built in `main.go` beside the pool and shared by the ObjectStore, the legacy `StorageImpl`, the cleanup handler and the data migration; `Close()` returns the connection before `Pool.Close` (K-5). | +| Checkpointing | `PRAGMA wal_autocheckpoint=0` on **every** pool connection (`file.PoolOptions.DisableAutoCheckpoint`, K-3); a supervised background goroutine runs `PRAGMA wal_checkpoint(PASSIVE)` when the `-wal` file exceeds a threshold after a gated commit, and on a timer. | +| Ownership guard | The legacy default `StorageImpl` gets `SetForeignKinds(file.IsContainerProfileKind)`: every full-object operation on a containerprofile key (`get`'s payload branch, fullSpec list, `getListWithSpec`, `appendGobObjectFromFile`, `delete`, `CreateWithConn`, `GuaranteedUpdateWithConn`, the single-writer create/update) returns an `InternalError` **before** touching a row, a file or a self-repair delete. Metadata-only reads are served. | +| `GeneratedNetworkPolicyStorage` | Its full-spec ContainerProfile list reads through the `containerprofiles` resource's own `storage.Interface` (the ObjectStore under the flag; the processor-wired legacy instance otherwise), never the default instance whose `get()` deletes the shared row of a key without a file. Its `knownservers` read stays on the default instance. | +| Cleanup | `ContainerProfileKind` is never in the generic relevancy walk's handler map (`initResourceToKindHandler`): the ContainerProfile arm — `deleteByTemplateHashOrWlid` plus, with relevancy on, the two missing-annotation handlers (`ResourcesCleanupHandler.ContainerProfileHandlers`) — runs from `ContainerProfileProcessor.cleanup()` only. Under the flag the handler carries the ObjectStore (`SetContainerProfileStore`, wired in `apiserver.go`), enumerates the namespace's **rows** and reclaims through `ObjectStore.Delete` (one gated transaction over the three tables, `Deleted` dispatched after). No CP file is read, written or removed. Flag-off keeps the file walk. | + +Flag off keeps the legacy store everywhere, without the backend's pragma, ownership guard, +write gate or startup migration. The rollback-safety checks described below still apply. + +## Data migration (`file.MigrateContainerProfiles`) + +Runs synchronously in `main.go` at every start with the flag on, after the pool and the gate are +built and **before** the cleanup goroutine and the API server (design §8.2, R2). Every batch is +one gated `BEGIN IMMEDIATE … COMMIT` (50 objects by default) containing only SQL on bytes prepared +before the ticket: the gob decode of a legacy file (the external `/usr/bin/migration` tool as the +fallback on a gob type mismatch — the last time it runs for this kind), and the JSON encode, run +on a pool connection released before the ticket. A PASSIVE checkpoint follows the last batch. +Legacy `.g` files are **left in place** (§8.4: they are the rollback). + +The reconcile of steps 2 and 2b runs on **every** start (R3); the done-flag in `migration_state` +gates only the file sweeps (steps 3–4). + +| Step | Predicate | Shape (`storage_cp_migration_total{shape,source}`) | Action | +|---|---|---|---| +| 2 | `metadata` row of kind `containerprofile` with `rv IS NULL` **or** no `payloads` row | `migrated` | file decoded, row and file agree on `resourceVersion`: metadata JSON rewritten with `rv`/`uid`, payload inserted | +| | | `diverged` | row and file disagree on `resourceVersion` (a PC-DIV shape, met once): the object gets `max(rowRV, payloadRV)+1`; PreSave's non-TS revert applied (a Completed row beats a Learning file) | +| | | `row_without_file` | no `.g` file: the row is deleted (today's `get()` self-repair, done once) | +| | | `legacy_rewrite{source=file}` | `rv IS NULL` **with** a `payloads` row — a legacy writer replaced the row after a rollback (K-1). The legacy writer wrote its file before its row, so the **file** is its content and the `payloads` body is stale: the body is rebuilt from the file; `rv` is the row JSON's `resourceVersion` (no `+1`; when a crash left the file one ahead, the larger persisted version wins and the JSON is rewritten to match) | +| | | `legacy_rewrite{source=payloads}` | same, no file (a row-only legacy write): the `payloads` body is kept, re-stamped at `rv` | +| | | `undecodable` | neither gob nor the tool decoded the file: skipped and counted, **never deleted**, the file left for the export tool (PM-2) | +| 2b | `payloads` row of kind `containerprofile` with no `metadata` row | `orphan_payload` | a `repairDelete` self-repair on one of the 3 undecodable-file `get()` sites (a corrupt or unmigratable `.g` file) still leaves its `payloads` row behind — the read-time fallback below can't safely serve a stale body there, only the every-start reconcile can. The missing-file branch also leaves payloads behind for excluded time-series rows; only an `rv IS NULL`, non-time-series ContainerProfile receives payload cleanup there. See "Rollback safety: read-time fallback" below. An orphan that reaches ObjectStore insertion would conflict with the payloads UNIQUE constraint; startup reconciliation removes it first. Deleted. | +| 3 (done-flag) | `.g` file under `/data//containerprofile/` with no row | `file_without_row` | imported at the file's `resourceVersion` and UID | +| 4 (done-flag) | `*.g.t*` staging files | `temp_file` | removed (never committed) | + +Every non-`migrated` outcome is logged per key at Warning: this is where the field frequency of +the divergence shapes is measured. The done-flag is written only when the sweep met no undecodable +file, so such a file stays reported at every start until an operator acts. + +Idempotent (a repaired row no longer matches the predicate; the second start reconciles nothing) +and resumable (a crash — an error, a panic, or the process dying inside a batch — rolls that batch +back; the next start completes from the same predicate). A migration error is fatal at startup: +the backend never serves a half-reconciled store, and turning the flag off is the rollback. + +`config.Config.ContainerProfileMigrationDryRun` runs the reconcile in count-only mode (no gate, +nothing written, the done-flag untouched) and logs the counts a real run would produce — the census +§8.3 requires before the flag is turned on anywhere. It is refused together with the backend flag. + +## Rollback safety: read-time fallback (`.omc/plans/rollback-safety-guard.md`) + +Simply turning `ContainerProfileSqliteBackend` back off — no binary change, a config edit — is a +*different* hazard from the downgrade below, and closer at hand: the flag-off `StorageImpl` runs +in the *same*, current binary, and its `get()`'s missing-file self-repair used to delete the +metadata row of any key the ObjectStore had created or updated since the flip (no `.g` file was +ever written for it) — the object destroyed, not merely invisible, with zero confirmation and no +`cpexport` step forcing itself on the operator first. + +The same-binary guard changes the missing-file path as follows: + +1. **`get()` serves the object from its `payloads` body** whenever all four database conditions + hold: the metadata row exists, `rv IS NOT NULL` (the row is ObjectStore- or migration-owned), + `is_time_series = 0`, and a `payloads` row exists. Every legacy write nulls `rv`/`uid` via + `INSERT OR REPLACE`, so the fallback excludes rows a legacy writer has touched since the flip, + including a `.g` file whose rename was lost to a crash. `ResourceVersion`/`UID` are stamped + from the metadata row's columns, using the same conversions as `cpexport`. +2. **Inspection failures preserve data.** A database query or payload decode failure returns a + non-NotFound error and retains both rows. A positively ineligible result can proceed to + metadata self-repair; it is distinct from an inspection failure. +3. **Payload cleanup is narrowly scoped.** The shared `repairDelete` still deletes metadata only. + Only the missing-file branch also deletes a ContainerProfile's payload when inspection + establishes an `rv IS NULL`, non-time-series row. Corrupt-file reads, migration-tool failures + and time-series rows keep their previous metadata-only repair behavior. +4. **`Create` protects fallback-visible objects**, including with `singleWriterEnabled=false`: + its database check prevents a missing legacy file from allowing an existing object to be + overwritten. The single-writer commit path retains its own metadata existence recheck. +5. **`Delete` propagates payload cleanup failures.** `deleteLocked` deletes payloads before + metadata and returns a `DeletePayloads` error to the caller instead of reporting success. + +The three undecodable-`.g`-file repair sites remain outside this fallback: a legacy write may +have superseded the SQLite body, so a corrupt or unmigratable file still triggers the existing +metadata-only self-repair. Existing but stale `.g` files are also out of scope: file reads still +win over SQLite payloads. The fallback does not make those files current. + +**Advisory-only, not a substitute for `cpexport`**: at every flag-off startup, a bounded census +(`LogFallbackEligibleContainerProfilesCensus`, its own `context.WithTimeout`, never `Fatal`) logs +a Warning with the count and example keys satisfying the four database conditions. These are +**database candidates**, not a count of objects actually served by the fallback: the census +neither checks for a missing `.g` file nor decodes the payload. Zero is logged at Info. A query +error is logged distinctly from a genuine zero count and never blocks startup. + +**Flag-off cleanup limitation**: cleanup walks `.g` files, so it never visits live fallback-only +records that have metadata and a payload but no file. Those objects remain readable through the +fallback, but are not reclaimed by that cleanup walk. Orphaned payloads (without metadata) are +inert — never served or resurrected — and also remain until the flag is re-enabled and migration +reconcile sweeps them (§8.2 step 2b, `orphan_payload`, above). + +## Rollback: `cpexport`, then downgrade (§8.4) + +The read-time fallback above only helps the same-binary flag-off case. An **older storage +binary** — a real image downgrade — opens the migrated database without error, but it reads only +the metadata row and the `.g` file: every key the ObjectStore created since the flip has **no +file**, every key it updated has a **stale** one, and the old binary's `get()` **deletes the +metadata row** of a key whose file is missing (its self-repair) — the object is destroyed, not +merely invisible. Its `INSERT OR REPLACE` also nulls `rv`/`uid`; the every-start reconcile repairs +that on re-enable (`legacy_rewrite`), but nothing repairs a deleted row. +`TestExport_DangerWithoutExport_OldBinaryDestroysNewStoreRows` pins all three symptoms. + +So the rollback order is fixed: **export first, downgrade second.** `/usr/bin/cpexport` +(`cmd/cpexport`; `file.ExportContainerProfiles`) writes every migrated row's payload back as the +legacy gob file at its key, staged and renamed exactly as the legacy writer does, at the row's +`resourceVersion` and UID. Rows with `rv IS NULL` (never migrated, or already rewritten by a legacy +writer) are skipped — their file is already the legacy writer's. The database is not touched: the +old binary ignores `rv`, `uid` and `payloads`; a row it never rewrites stays consistent for the +re-enable, and a row it rewrites or deletes is exactly the `legacy_rewrite` / `orphan_payload` +shape the reconcile repairs. Run it **with the server stopped** (scale to 0, run against the PVC), +never beside a serving process; `-dry-run` counts without writing. + +``` +cpexport [-root /data] [-db /data/metadata.sq3] [-dry-run] +``` + +The round trip is behaviourally identical, not byte-identical +(`TestExport_RoundTripIsBehaviourallyIdentical`): gob encodes maps (labels, annotations) in Go's +randomised map order, so two encodes of one object differ byte-wise, and the JSON codec keeps +`creationTimestamp` to the second and empty collections as empty (divergences 4 and 6 above). +Everything the old binary reads back — every field, the `resourceVersion`, the UID — is asserted +equal to what it served before the migration, and the full cycle export → downgrade → old-binary +writes and deletes → re-enable is asserted lossless (`TestExport_ThenDowngradeThenReEnable`). + +**Deployment precondition (R12):** one storage pod at a time (the chart's `replicas: 1` + +`strategy: Recreate`); nothing in the database fences an older binary. + +## Metrics + +`storage_sqlite_write_hold_seconds{path}` (the migration's batches are `path="cp_migration"`), +`storage_write_gate_wait_seconds{priority}`, `storage_sqlite_busy_wait_seconds`, +`storage_cp_cas_conflict_total{op}`, `storage_cp_ownership_refusal_total{op}`, +`storage_cp_migration_total{shape,source}`, `storage_sqlite_wal_pages`, +`storage_sqlite_freelist_count`, `storage_sqlite_checkpoint_total{outcome}`, +`storage_sqlite_ungated_write_total{op,table}` (the AC-G1 production canary; alert at > 0). Gate +commits are also counted under the existing `storage_single_writer_commit_total`. + +## Known, intended divergences from the legacy store + +The differential suites (`pkg/registry/file/sqliteobject_differential_test.go`, +`pkg/registry/softwarecomposition/containerprofile/backend_differential_test.go`) assert these as +*what* differs, and that nothing else does: + +1. **AfterCreate crash atomicity** — a TS create whose `time_series` write fails leaves an object + without a series row on the legacy store; nothing on the new one. +2. **TS admission after base completion** — a base that becomes Completed/Full between PreSave's + read and the commit is admitted by the legacy store and refused by the new one. +3. **Rowid-stable pagination** — the legacy `INSERT OR REPLACE` re-inserts an updated row, so a + paginating LIST sees it again; the new `UPDATE` keeps the rowid (continue tokens of later rows + differ by one per such re-insert). +4. **`creationTimestamp` precision** — gob keeps nanoseconds, JSON (RFC 3339) keeps seconds. +5. **R4 per-TS CAS** — a TS object updated during a tick is silently deleted by the legacy store; + the new store conflicts once, retries and merges the update. +6. **nil vs empty collections** (found by the suite, not in the design's list) — `v1beta1` spec + collections without `omitempty` (`execs`, `opens`, `capabilities`, `endpoints`, …) decode as + empty slices from JSON and as nil from gob, so REST renders `[]` where the legacy store renders + `null`. + +## Tests + +`sqliteobject_migration_test.go` drives a real legacy `StorageImpl` (no guard, no gate, a second +pool on the same database file) as the "old binary" and the gated pool + ObjectStore as the "new +binary": the plain migration (content, RV and UID preserved, files kept, CAS live afterwards, +second start a no-op); every reconcile shape with its own fixture; the R3/K-1 rollback cycle (the +stale-body and permanent-conflict symptoms before the reconcile, the repair from the file after +it, `rv == json resourceVersion`, the next CAS succeeding) and its row-only variant; the K-2 +orphan (the UNIQUE failure on `Create` before, success after); resumability under an injected +error, a panic, and a killed child process (`os.Exit` inside the second batch, real files, the +parent resumes); the dry run's counts predicting the real run and writing nothing; and that a +batch waits its turn behind another gate holder. `sqliteobject_cleanup_test.go` and +`sqliteobject_gnp_test.go` pin the cleanup and GNP re-pointing; all of them fail on the previous +wiring. The X-A frozen gate (Lane 0) is not on this branch's base; INV-3 is asserted as parity +between the backends. diff --git a/docs/features/storage-lock-pool-metrics.md b/docs/features/storage-lock-pool-metrics.md index e64b20d0b..6caacc510 100644 --- a/docs/features/storage-lock-pool-metrics.md +++ b/docs/features/storage-lock-pool-metrics.md @@ -40,10 +40,19 @@ endpoint was added). `lockTimeout`/`poolTimeout` backstop and the caller received a `ServerTimeout`). Implementation lives in `pkg/metrics/metrics.go` (`ObserveLockWait`, `ObservePoolWait`); -call sites are in `pkg/registry/file/storage.go` at each `s.locks.Lock`/`RLock` and -`s.pool.Take` acquisition. The existing Debug-level `lockDuration > 1s` log lines are left -in place — they remain useful for correlating a specific slow request with its key, which -the aggregate histograms can't do. +call sites are in `pkg/registry/file/storage.go` and `singlewriter.go` at each +`s.locks.Lock`/`RLock` and `s.pool.Take` acquisition, plus the two consolidation-side takes +that originally went unobserved: `ContainerProfileStorageImpl.WithConnection` (the +listing connection and every consolidation worker's connection, `kind=containerprofiles`) +and `createSingleWriter`'s `AfterCreate` connection. The only `Take` still outside the +histogram is `ResourcesCleanupHandler.CleanupTask`'s (`cleanup.go`). The existing +Debug-level `lockDuration > 1s` log lines are left in place — they remain useful for +correlating a specific slow request with its key, which the aggregate histograms can't do. + +The histogram's sample count is also how the work-budget test +(`docs/features/storage-measurement-harness.md`, Tier A) counts pool takes per scenario; a +per-key lock's read/write mode, which these histograms do not label, is reported to that +test through the nil-in-production `utils.SetLockObserver` hook in `pkg/utils/mutex.go`. ### Consolidation counters (completed-immutability and divergence) diff --git a/docs/features/storage-measurement-harness.md b/docs/features/storage-measurement-harness.md new file mode 100644 index 000000000..2adaba8eb --- /dev/null +++ b/docs/features/storage-measurement-harness.md @@ -0,0 +1,208 @@ +# Storage measurement harness (work budgets and paired A/B) + +> **Backend A/B.** Tier B can compare the two ContainerProfile backends on the +> same commit: `PERF_AB_BACKEND=objectstore` makes a round drive the SQLite-native +> `ObjectStore` (`config.ContainerProfileSqliteBackend`, see +> `containerprofile-sqlite-backend.md`) instead of the legacy `StorageImpl`. The +> backend is provenance in the round JSON (`backend`), not part of the effective +> config, so `BASE=HEAD PERF_AB_BASE_ENV="PERF_AB_BACKEND=legacy" +> PERF_AB_HEAD_ENV="PERF_AB_BACKEND=objectstore" hack/perf-ab.sh` is a valid A/B. +> The round also has a `list` client class (`list-p99-ms`, headline; the design's +> PM-1 detector) and records `write-bytes` from `/proc/self/io` (info). + +## Summary + +Two instruments for the `pkg/registry/file` hot paths, so that a change to +the storage layer is reviewed with a number and not only with a correctness +test: + +| Tier | Question | Mechanism | Runs | +|---|---|---|---| +| **A — work budgets** (`TestWorkBudget`) | Did this change add I/O, locks, statements or events to a hot-path call? | Count operations per scenario; compare **exactly** against a checked-in golden | Always, inside `go test ./...`, ~1 s | +| **B — paired A/B** (`make perf-ab`) | Under contention, is HEAD slower or lower-throughput than its merge-base by more than X%? | Two test binaries (merge-base, HEAD) interleaved on the same machine in one window; a relative verdict with its own noise estimate | Before SHIP on any hot-path PR; CI on demand and nightly | + +Neither tier checks in an absolute latency. Tier A measures *change*, Tier B +measures *whether the change matters under contention*. + +## Tier A — work budgets + +`pkg/registry/file/workbudget_test.go` runs eight single-goroutine scenarios +on a fresh storage each (pool 10, one consolidation worker, in-memory payload +files, fixtures cloned from `testdata/p1.json` and stamped at run time): + +| # | Scenario | Hot path | +|---|---|---| +| S1 | Learning tick: Learning base, three new reports with data | `ConsolidateTimeSeries` → `consolidateKeyTimeSeries` → merge → save → per-row delete | +| S2 | Empty tick: nothing pending | the listing only | +| S3 | Frozen tick: Completed/Full base, two late reports with data | the frozen arm | +| S4 | Divergent tick: payload Completed/Full at RV n+1, metadata row Learning at RV n, plus S3's late reports | the heal arm, then S3 | +| S5 | REST `Get` of a base key | `acquireLockedConn` read path | +| S6 | REST `Create` of a TS profile | `PreSave` + single-writer commit + `AfterCreate` | +| S7 | REST `GuaranteedUpdate` whose `tryUpdate` is a no-op | `guaranteedUpdateSingleWriter` short-circuit | +| S8 | One processed TS profile deleted as the consolidation pass deletes it | `deleteProcessedTimeSeries` | + +Five counters, each a before/after delta around the scenario only: + +1. **SQL statements by Go call site** — every `sqlitex.Execute` in + `sqlite.go` (13 sites, one per function) plus the transaction openers + (`Transaction`, `Save:saveObject`, `Save:commit`) report through + `observeStmt(site)` in `workbudget.go`. The observer is an + `atomic.Pointer[func(string)]`: nil in production (one load and a nil + check, no allocation), installable without a data race while a leaked + goroutine from another test is still executing statements. +2. **Per-key lock acquisitions by mode** — `utils.SetLockObserver` in + `pkg/utils/mutex.go`, the same nil-in-production shape, reports every + `Lock`/`RLock` with its outcome. The golden's `lock` is + `{rlock, lock, timeout}`. +3. **Pool takes** — the sample-count delta of the existing + `storage_pool_wait_duration_seconds` histogram. This PR made the two + previously unobserved take sites report to it: + `ContainerProfileStorageImpl.WithConnection` (the listing and every + consolidation worker) and `Create`'s `AfterCreate` connection. + `cleanup.go`'s `CleanupTask` take is still unobserved and outside every + scenario. +4. **Payload-file operations** — a counting `afero.Fs` (`open`, `rename`, + `remove`). One payload read is one open; one save is one open plus one + rename. +5. **Watch events** — a watcher on `/`, drained with a deadline after the + scenario. + +Scenarios run under a package mutex, never in parallel, and each ends with a +**settle check**: the observers stay installed for 50 ms after the scenario +returns, then are cleared and read once more under their mutex; any late +observation (statement, lock, file op, or a moved pool histogram) fails the +test as `contaminated: ` instead of a flaky golden mismatch. Every +scenario pins `singleWriterEnabled == true` at its start. Hook closures never +call `testing.T`. + +The golden is `pkg/registry/file/testdata/workbudget.golden.json`; comparison +is exact equality with the diff printed per field. Regenerate with + +``` +go test ./pkg/registry/file -run TestWorkBudget -update +``` + +**A golden change is a review item.** The commit that changes it states each +delta and why; the reviewer, not the author, checks the attribution. Four +invariants hold regardless of the golden: + +1. a no-op writes nothing (S2, S7: no rename, no `WriteJSON`, no events); +2. a frozen tick writes nothing to the base (S3: no rename, no `WriteJSON`, + no `Modified`) — enabled by `frozenTickInvariant` once the frozen gate + (kubescape/storage#399) is in the tree; +3. one REST read is one read lock and one connection (S5); +4. the single-writer `ROLLBACK` recovery never fires on a passing path. + +## Tier B — paired A/B + +`hack/perf-ab.sh` (`make perf-ab`) compares HEAD against `BASE` (default +`git merge-base origin/main HEAD`) with `PAIRS` (default 10) interleaved +rounds. Per round, `TestPerfABRound` in `containerprofile_load_test.go` +runs a **fixed amount of work** with **closed-loop clients** on the +production shape — pool 10, 8 shards, `Workers = 2`, `GOMAXPROCS=8`, 6 +writers × 2400 Creates, 25 readers × 12000 Gets, 3 updaters × 1200 +GuaranteedUpdates, 12 base keys, consolidation ticking every 250 ms plus +three ticks after the clients finish — and writes one JSON: per-class +latency percentiles, wall time and throughput, the six process-registry +series (`storage_lock_wait_duration_seconds`, +`storage_pool_wait_duration_seconds`, +`storage_single_writer_queue_wait_duration_seconds`, +`storage_single_writer_commit_total`, +`storage_single_writer_conflict_retry_total`, +`storage_single_writer_queue_depth` max), and an `effective` block read back +from the constructed objects (probed pool size, `len(shards)`, `Workers`, +`GOMAXPROCS`, `singleWriterEnabled`, op counts, the pinned collapse-settings +TTL). It also prints +`BenchmarkPerfAB/` lines for `benchstat`. + +The driver: + +1. refuses to start if the 1-minute load average exceeds `nproc/2` + (`PERF_AB_ALLOW_NOISY=1` overrides and stamps the verdict `(noisy)`); +2. builds `base.test` from a worktree at `BASE` **with HEAD's harness files + overlaid** (the load test and the thresholds), so a base that predates the + harness runs the same instrument; a base that cannot host it exits 2 with + the build error, not a bare compile failure; +3. pins both binaries to the first `nproc/2` distinct cores with `taskset` + and `GOMAXPROCS=8`; +4. runs three base-only probe rounds and aborts (exit 3) if any headline + metric's CV exceeds 50 % — an early abort, not the control; +5. interleaves `PAIRS` rounds of base, head on fresh temp databases, + sampling the load average per round into `schedule.txt`; a pair whose + sample exceeds `nproc/2` is marked, and more than `⌈PAIRS/3⌉` marked + pairs is INCONCLUSIVE regardless of the statistics; +6. computes the verdict with `hack/perfab` from the pre-registered thresholds + in `pkg/registry/file/testdata/perfab.thresholds.json`. + +Per headline metric (REST Get p99, Create p99, GuaranteedUpdate p95, tick +p50/p99, ops/s) the primary statistic is a **paired t-test on the per-round +log-ratios** `d_i = ln(head_i/base_i)`; Mann–Whitney U on the two samples is +the second opinion. REGRESSION needs the mean ratio past the threshold +(+20 % on latencies, +30 % on tick p99, −10 % on throughput) **and either** +test at p < 0.05; a row where the two tests disagree is flagged `SPLIT`. +Timeout counts regress when HEAD > BASE significantly; the conflict rate +when it rises by more than 5 points; `errOther`, `>5 s` and commit panics are +**hard** rows — any on HEAD when BASE has none, no statistics. GuaranteedUpdate +is gated on p95: under the pinned shape about 1 % of updates hit +`acquireLockedConn`'s 250 ms connection-attempt cliff, so its p99 straddles the +cliff and is bimodal round to round; it is reported (`info`) but never gates, +and the cliff itself is the `pool-wait-timeouts` row. `tick-total-s` (the +sum of all consolidation passes in the round, for a fixed number of rows) is +reported alongside tick p50/p99 because with fixed-interval ticking a slower +pass accumulates more rows for the next one, so per-pass percentiles partly +measure rows-per-pass; the total is the per-row cost. Noise is +measured post hoc from the pairs: the paired CV is the SD of `d_i` (over 25 % +on a headline metric is INCONCLUSIVE), and `MDE = (t_{N-1,0.975} + +t_{N-1,0.8}) · s_d / √N`; a metric whose MDE exceeds its threshold is +UNDERPOWERED with the `N'` needed printed. + +Exit codes and verdict lines (quote the line verbatim in the PR): + +| Exit | Line | +|---|---| +| 0 | `PASS` | +| 1 | `REGRESSION t=

mw=

` | +| 2 | `CONFIG MISMATCH ` — the arms did not run the same workload (or base cannot host the harness) | +| 3 | `INCONCLUSIVE paired-CV=%` / `INCONCLUSIVE load-marked=/` / `INCONCLUSIVE probe …` | +| 4 | `UNDERPOWERED N'=` | + +Precedence when several apply: config mismatch, load-marked, hard rows, +paired CV, statistical regression, underpowered, pass. + +Artifacts (rounds, logs, `schedule.txt`, `verdict-table.txt`, `verdict.txt`) +land in `.omc/artifacts/perf-ab//` unless `PERF_AB_OUT_DIR` is +set. `.github/workflows/perf-ab.yaml` runs the same driver on +`workflow_dispatch`, nightly against the previous day's `main`, and on PRs +labelled `perf`; runners are noisy, so INCONCLUSIVE is expected often there +and is reported in the check summary, never hidden — the local run on a quiet +machine is the authoritative one. Only a REGRESSION fails the check. + +Two harness isolation choices are pinned per round and echoed in `effective`: +the SQLite busy timeout is 5 s (production: 60 s), and `collapseSettingsTTL` +is one hour (production: 10 s). The latter because a consolidation save that +has already written refreshes the CollapseConfiguration cache on a second +connection, and with no CR present `get()`'s `DeleteMetadata` on that +connection waits on the write lock the same goroutine holds — a self-deadlock +resolved only by the busy timeout, during which every shard commit waits too. +Whether a round crosses a TTL boundary is wall-clock phase, not the change +under test; the stall itself shows up in the `over-one-sec` row of an +unpinned run. + +`TestContainerProfileLoad` (`LOAD_TEST=1`) remains as a time-boxed +diagnostic with env tunables; its absolute numbers are not evidence. + +## The rule this adds + +For any change under +`pkg/registry/file/{storage,singlewriter,containerprofile_*,sqlite}.go`: + +``` +go build ./... +go test ./... # includes TestWorkBudget +go test -race -count=20 ./pkg/registry/file/ -run 'Consolidate|SingleWriter|Delete' +make perf-ab BASE=$(git merge-base origin/main HEAD) # quote the verdict line +``` + +A golden change is stated per row in the commit message; the verdict line is +quoted in the PR description; SHIP is not written against INCONCLUSIVE or +UNDERPOWERED without a second run on a quieter machine or at `N'`. diff --git a/docs/features/write-gate-sharing.md b/docs/features/write-gate-sharing.md new file mode 100644 index 000000000..c8f9fa114 --- /dev/null +++ b/docs/features/write-gate-sharing.md @@ -0,0 +1,109 @@ +# Write-gate sharing: one SQLite writer for every kind + +**Status:** implemented behind `containerProfileSqliteBackend` (R0 + R1 of +`.omc/plans/write-gate-sharing.md`); the flag flip (R2) and the shard removal +(R3) are separate, later steps. + +## The bug class + +SQLite has one write lock per database. Under the ContainerProfile SQLite +backend every ObjectStore write goes through a *write gate*: a FIFO ticket +queue in Go owning one dedicated connection, `BEGIN IMMEDIATE … COMMIT`. Every +write that does **not** go through the gate acquires the same lock through +SQLite's busy handler, which polls every 1…100 ms, is not `ctx`-bounded, and +loses systematically against a gate that commits continuously. Such a writer +stalls for the whole busy timeout (60 s in production) and the gate's own +histograms never see it — the loser is on a pool connection the gate does not +instrument. + +Two instances were found and point-fixed before this change (the collapse +settings self-stall, #401; the SBOM self-repair stall, `df50b1e3`). Both were +the same `DELETE FROM metadata` reached two ways. Nine more sites remained +(`write-gate-sharing.md` §1, W1–W9b): the shard commit, `saveObject`, +`deleteLocked`, `get()`'s four self-repair deletes, the three gob-migration +rewrites and the cleanup tick's row delete and sidecar migration. + +## What changed + +- **One gate per process.** `main.go` builds the gate beside the pool when the + flag is on (`file.NewWriteGate`) and hands it to the ObjectStore, the legacy + `StorageImpl` (`SetWriteGate`) and the cleanup handler. The apiserver's + pre-shutdown hook closes the store, then the gate, before `Pool.Close`. + With the flag off no gate exists and every site runs today's code, byte for + byte (Tier A goldens unchanged). +- **One helper at nine sites.** `gatedWrite` (`storage.go`) is the only place + the flag-off/flag-on branch is chosen: bare statements or today's savepoint + on the caller's connection without a gate; `gate.run` with one. + - W1 `singleWriter.commitGated`: the shard holds **no pool connection** + while queued; the CAS read joins the gated transaction. + - W2 `saveObject(ctx, conn, …, priority, path)`: the row + rename. + - W3 `deleteLockedGated`: `DELETE … RETURNING` captured raw, decoded after + release; `Delete` takes `Lock(key)` only. + - W4–W7a `repairDelete`; W6b/W7b/W8 through W2 (`migrate` label). + - W9a/W9b `ResourcesCleanupHandler.write` on the tick's `ctx`. +- **The gate is not re-entrant, and says so.** `fn` receives a ctx marked as + gate-held; a nested acquire through it is refused at O(1) + (`storage_write_gate_reentrant_total`; a panic under the test binary). A + watchdog logs a hold that outlives `gateWatchdogThreshold` with every + goroutine's stack (`storage_write_gate_hold_age_seconds`). +- **One gate per pool, enforced.** `newWriteGate` refuses a pool that already + has a live gate (`writegate_registry.go`). +- **Refused configuration.** `containerProfileSqliteBackend` without + `singleWriterEnabled` is fatal at startup and `ErrGateRequiresSingleWriter` + at runtime: with the single writer off every REST write would queue on the + gate holding a pool connection. + +### The connection rule and where it is relaxed + +Hot paths hold no pool connection while queued (the shard commit, `Delete`). +Cold paths — a repair from `Get`, a migration rewrite, the cleanup walk — keep +the connection they already hold; the bound is measured, not assumed +(`TestTG5_…`, `storage_pool_wait_duration_seconds{outcome="timeout"}` = 0). + +## The invariant, not the enumeration + +The proof is not that W1–W9b are fixed; it is that the next site fails CI: + +- **AC-G1** (`main_test.go`, `writegate_registry.go`): every pool connection + carries an authorizer that reports each INSERT/UPDATE/DELETE prepared on it. + In production it counts `storage_sqlite_ungated_write_total{op,table}` when + the pool has a gate that never owned the connection — the R2 canary, alert + at > 0. Under the test binary every such statement is recorded with its + stack and judged at each gated environment's cleanup (and once more at + exit): a write on a connection no gate of the pool ever owned fails the + test naming the site. Fixtures seed state through a non-pool handle + (`openFixtureConn`); the recorder is never windowed off (a cached statement + re-executes without the authorizer). +- **AC-G2** (`writegate_acg2_test.go`, `writegate_acg2_on_test.go`): read + entry point × key state, per topology. Ordinary reads complete while the + writer is held. Flag-off repairs attempt a write but leave metadata unchanged + under the held SQLite lock; flag-on repairs queue behind the gate and complete + after explicit release. Generous deadlines detect deadlocks; elapsed times + are diagnostic measurements, not scheduler-sensitive pass/fail thresholds. +- **T-G1** (`writegate_tg1_test.go`): one env per site, W1–W9b. + +## Metrics + +| Series | Meaning | +|---|---| +| `storage_sqlite_write_hold_seconds{path,kind}` | gate hold per transaction; legacy paths `legacy_commit`, `legacy_delete`, `repair`, `migrate`, `cleanup`, `cleanup_migrate` | +| `storage_sqlite_write_hold_step_seconds{path,step}` | the legacy commit's payload rename | +| `storage_sqlite_ungated_write_total{op,table}` | must stay 0 (AC-G1) | +| `storage_sqlite_busy_wait_seconds` | must stay ~0 with the flag on (AC-G3) | +| `storage_write_gate_reentrant_total{path}` | must stay 0 | +| `storage_write_gate_hold_age_seconds` | alert at > 5 s | + +## Measuring + +Tier B carries legacy traffic (`containerprofile_load_test.go`, harness +version 3): two writers cycling sbomsyft objects at `LOAD_LEGACY_KB` and small +vulnerability manifests through create → update → delete, and one cleanup tick +reclaiming `LOAD_CLEANUP_ROWS` rows, concurrent with the CP scenario. + +``` +PERF_AB_BACKEND=objectstore PERF_AB_OUT=round.json go test -run TestPerfABRound ./pkg/registry/file/ +``` + +reports `hold-p99-ms/`, `busy-wait-max-ms`, `ungated-writes`, +`pool-wait-timeouts` and the legacy classes' latencies next to the CP ones; +`hack/perf-ab.sh` compares two arms. diff --git a/go.mod b/go.mod index 6d21212e3..7ae7df5d8 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/kubescape/k8s-interface v0.0.214 github.com/ncw/directio v1.0.5 github.com/olvrng/ujson v1.1.0 + github.com/prometheus/client_model v0.6.2 github.com/puzpuzpuz/xsync/v2 v2.4.1 github.com/spf13/afero v1.15.0 github.com/spf13/cobra v1.10.2 @@ -43,6 +44,7 @@ require ( k8s.io/klog/v2 v2.130.1 k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 + pgregory.net/rapid v1.2.0 sigs.k8s.io/structured-merge-diff/v6 v6.3.0 zombiezen.com/go/sqlite v1.4.0 ) @@ -141,7 +143,6 @@ require ( github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.23.2 // indirect - github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect diff --git a/go.sum b/go.sum index bf06587cd..c55f4283e 100644 --- a/go.sum +++ b/go.sum @@ -1387,6 +1387,8 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/hack/perf-ab.sh b/hack/perf-ab.sh new file mode 100755 index 000000000..9d0c33283 --- /dev/null +++ b/hack/perf-ab.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# Tier B of the storage measurement harness: paired A/B of HEAD against its +# merge-base, rebuilt in the same run window on the same machine. +# Design: .omc/plans/raw-write-bypass-elimination.md, A.13.4. +# +# make perf-ab # BASE = git merge-base origin/main HEAD +# BASE= PAIRS=10 hack/perf-ab.sh +# +# Inputs (env): +# BASE commit to compare against (default: merge-base with origin/main) +# PAIRS interleaved (base, head) rounds (default 10) +# PROBE_ROUNDS base-only early-abort rounds (default 3) +# PERF_AB_OUT_DIR where rounds, logs, schedule.txt and verdict.txt go +# PERF_AB_ALLOW_NOISY 1 to run despite a high 1-minute load average (verdict stamped "(noisy)") +# PERF_AB_KEEP 1 to keep the base worktree and build artifacts +# PERF_AB_BASE_ENV extra KEY=VALUE pairs (space-separated) for the base arm's rounds +# PERF_AB_HEAD_ENV ... for the head arm's rounds. With BASE=HEAD this turns the A/B +# into a same-commit comparison of two configurations, e.g. +# PERF_AB_BASE_ENV="PERF_AB_BACKEND=legacy" +# PERF_AB_HEAD_ENV="PERF_AB_BACKEND=objectstore" +# LOAD_HOT_KEYS=1 in BOTH arms selects the same-key +# contention shape (every updater on base key 0); it is +# part of the effective config, so one arm alone is a +# CONFIG MISMATCH. +# +# Exit codes: 0 PASS, 1 REGRESSION, 2 CONFIG MISMATCH (or base cannot host +# HEAD's harness), 3 INCONCLUSIVE, 4 UNDERPOWERED. +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +PKG=pkg/registry/file +PAIRS=${PAIRS:-10} +PROBE_ROUNDS=${PROBE_ROUNDS:-3} +GOMAXPROCS_PIN=${GOMAXPROCS_PIN:-8} +THRESHOLDS=$PKG/testdata/perfab.thresholds.json +# Files overlaid onto the base worktree so both arms run HEAD's instrument. +HARNESS_FILES=("$PKG/containerprofile_load_test.go" "$THRESHOLDS") + +OUT=${PERF_AB_OUT_DIR:-$ROOT/.omc/artifacts/perf-ab/$(date +%Y%m%d-%H%M%S)} +mkdir -p "$OUT" +BASE_WT=$OUT/base-worktree + +cd "$ROOT" +HEAD_SHA=$(git rev-parse HEAD) +if [ -z "${BASE:-}" ]; then + BASE=$(git merge-base origin/main HEAD) +fi +BASE_SHA=$(git rev-parse --verify "$BASE^{commit}") + +log() { printf '%s %s\n' "$(date +%H:%M:%S)" "$*"; } +die() { echo "perf-ab: $*" >&2; exit "${2:-2}"; } + +cleanup() { + if [ "${PERF_AB_KEEP:-0}" != "1" ] && [ -d "$BASE_WT" ]; then + git -C "$ROOT" worktree remove --force "$BASE_WT" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +# 1. Start-of-run load check: an early abort, not the control (see step 5). +NPROC=$(nproc) +LOAD_BOUND=$(( NPROC / 2 )) +load1() { cut -d' ' -f1 /proc/loadavg; } +NOISY_FLAG="" +L1=$(load1) +if awk -v l="$L1" -v b="$LOAD_BOUND" 'BEGIN{exit !(l > b)}'; then + if [ "${PERF_AB_ALLOW_NOISY:-0}" = "1" ]; then + log "WARNING: 1-minute load $L1 exceeds nproc/2=$LOAD_BOUND; continuing (PERF_AB_ALLOW_NOISY=1), verdict will be stamped (noisy)" + NOISY_FLAG="-noisy" + else + echo "INCONCLUSIVE load=$L1 exceeds nproc/2=$LOAD_BOUND at start; refusing to run (PERF_AB_ALLOW_NOISY=1 overrides)" | tee "$OUT/verdict.txt" + exit 3 + fi +fi +log "load check: 1-minute load $L1, bound nproc/2=$LOAD_BOUND" + +# 2. CPU set from nproc: the first floor(nproc/2) distinct physical cores +# when lscpu can tell them apart, else the first floor(nproc/2) CPUs. +NCPU=$(( NPROC / 2 )) +[ "$NCPU" -ge 1 ] || NCPU=1 +if command -v lscpu >/dev/null 2>&1 && lscpu -p=CPU,CORE >/dev/null 2>&1; then + CPUSET=$(lscpu -p=CPU,CORE | grep -v '^#' | awk -F, '!seen[$2]++ {print $1}' | head -n "$NCPU" | paste -sd, -) +else + CPUSET="0-$(( NCPU - 1 ))" +fi +PIN=(env "GOMAXPROCS=$GOMAXPROCS_PIN") +if command -v taskset >/dev/null 2>&1; then + PIN=(taskset -c "$CPUSET" env "GOMAXPROCS=$GOMAXPROCS_PIN") +fi +BASE_ENV=${PERF_AB_BASE_ENV:-} +HEAD_ENV=${PERF_AB_HEAD_ENV:-} +log "head=$HEAD_SHA base=$BASE_SHA pairs=$PAIRS probe=$PROBE_ROUNDS cpuset=$CPUSET GOMAXPROCS=$GOMAXPROCS_PIN out=$OUT base_env='$BASE_ENV' head_env='$HEAD_ENV'" + +# 3. Build both binaries with the same harness: HEAD's harness files are +# overlaid onto the base worktree before `go test -c`. +git worktree add --detach "$BASE_WT" "$BASE_SHA" >/dev/null 2>&1 || die "git worktree add $BASE_SHA failed" +for f in "${HARNESS_FILES[@]}"; do + mkdir -p "$BASE_WT/$(dirname "$f")" + cp "$ROOT/$f" "$BASE_WT/$f" +done +log "building base.test (base $BASE_SHA + HEAD harness overlay)" +if ! (cd "$BASE_WT" && go test -c -o "$OUT/base.test" "./$PKG" >"$OUT/build-base.log" 2>&1); then + { + echo "CONFIG MISMATCH base $BASE_SHA cannot host HEAD's harness (${HARNESS_FILES[*]}):" + grep -v '^#' "$OUT/build-base.log" | head -n 5 + } | tee "$OUT/verdict.txt" + exit 2 +fi +log "building head.test" +(cd "$ROOT" && go test -c -o "$OUT/head.test" "./$PKG" >"$OUT/build-head.log" 2>&1) || { cat "$OUT/build-head.log"; die "head build failed"; } +(cd "$ROOT" && go build -o "$OUT/perfab" ./hack/perfab) || die "perfab tool build failed" +PERFAB=$OUT/perfab + +SCHEDULE=$OUT/schedule.txt +: >"$SCHEDULE" +: >"$OUT/base.txt" +: >"$OUT/head.txt" + +# run_round

.g.t, then a rename into

.g. +func writeLegacyPayloadFile(fs afero.Fs, p string, obj runtime.Object) error { + if err := fs.MkdirAll(filepath.Dir(p), 0755); err != nil { + return fmt.Errorf("mkdir: %w", err) + } + finalPath := makePayloadPath(p) + tmpPath := finalPath + ".t" + f, err := openPayloadFileWithFallbackFs(fs, tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("open payload file: %w", err) + } + w := NewDirectIOWriter(f) + if err := gob.NewEncoder(w).Encode(obj); err != nil { + _ = w.Close() + _ = f.Close() + _ = fs.Remove(tmpPath) + return fmt.Errorf("encode payload: %w", err) + } + if err := errors.Join(w.Close(), f.Close()); err != nil { + _ = fs.Remove(tmpPath) + return fmt.Errorf("close payload file: %w", err) + } + if err := fs.Rename(tmpPath, finalPath); err != nil { + _ = fs.Remove(tmpPath) + return fmt.Errorf("rename payload into place: %w", err) + } + return nil +} + +// openPayloadFileWithFallbackFs is StorageImpl.openPayloadFileWithFallback +// without the receiver. +func openPayloadFileWithFallbackFs(fs afero.Fs, path string, flag int, perm os.FileMode) (afero.File, error) { + f, err := fs.OpenFile(path, openFlagDirect|flag, perm) + if err != nil && isDirectIOUnsupported(err) { + f, err = fs.OpenFile(path, flag, perm) + } + return f, err +} diff --git a/pkg/registry/file/sqliteobject_export_test.go b/pkg/registry/file/sqliteobject_export_test.go new file mode 100644 index 000000000..813f6baa6 --- /dev/null +++ b/pkg/registry/file/sqliteobject_export_test.go @@ -0,0 +1,476 @@ +package file + +// The reverse export (§8.4: export-then-downgrade) and the danger it closes. + +import ( + "bytes" + "context" + "encoding/gob" + "fmt" + "path/filepath" + "sync/atomic" + "testing" + "time" + + helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + "k8s.io/apiserver/pkg/storage" + "zombiezen.com/go/sqlite/sqlitex" +) + +func (e *migrationEnv) export(opts ContainerProfileExportOptions) *ContainerProfileExportReport { + e.t.Helper() + report, err := ExportContainerProfiles(e.ctx, e.pool, e.fs, DefaultStorageRoot, e.scheme, opts) + require.NoError(e.t, err) + return report +} + +func (e *migrationEnv) storeCreate(name string) *softwarecomposition.ContainerProfile { + e.t.Helper() + out := &softwarecomposition.ContainerProfile{} + require.NoError(e.t, e.store.Create(e.ctx, e.key(name), e.plain(name), out, 0)) + return out +} + +func (e *migrationEnv) readFile(name string) []byte { + e.t.Helper() + b, err := afero.ReadFile(e.fs, e.filePath(name)) + require.NoError(e.t, err) + return b +} + +func decodeGob(t *testing.T, b []byte) *softwarecomposition.ContainerProfile { + t.Helper() + obj := &softwarecomposition.ContainerProfile{} + require.NoError(t, gob.NewDecoder(bytes.NewReader(b)).Decode(obj)) + return obj +} + +// The danger, on this branch's code, without the export: an older binary +// opening the migrated database destroys the rows the ObjectStore wrote and +// nulls rv/uid on the rows it rewrites. +func TestExport_DangerWithoutExport_OldBinaryDestroysNewStoreRows(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + e.seed(2) + updated, created := e.key("plain-00"), e.key("created-under-the-new-store") + e.startNew() + e.mustMigrate(ContainerProfileMigrationOptions{}) + createdObj := e.storeCreate("created-under-the-new-store") + require.NoError(t, e.storeUpdate(e.ctx, updated)) + require.Equal(t, "3", e.mustStoreGet(updated).ResourceVersion) + e.assertINV2(updated, created) + exists, err := afero.Exists(e.fs, e.filePath("created-under-the-new-store")) + require.NoError(t, err) + require.False(t, exists, "a key created under the new store has no file") + + // Downgrade without the export: the old binary runs. + e.stopNew() + + // (1) A key with no file: get() used to delete the metadata row and + // destroy the object. Rollback-safety guard Part 2: this key satisfies + // all four fallback conditions (a metadata row, rv non-NULL, not a + // time-series row, a payloads row), so the read is now served from the + // payloads body at the row's rv/uid instead — no self-repair, no side + // effects. The remaining dangers below, (2) and (3), are unaffected. + servedCreated, err := e.storeGetLegacy(created) + require.NoError(t, err, "Part 2: the fallback serves this key instead of destroying it") + require.Equal(t, createdObj.ResourceVersion, servedCreated.ResourceVersion, "stamped from the row's rv column") + require.Equal(t, createdObj.UID, servedCreated.UID, "stamped from the row's uid column") + row := e.inspect(created) + require.True(t, row.metaExists, "Part 2: no self-repair fires on a fallback-served read") + require.True(t, row.payloadExists, "Part 2: the payloads row the read was served from survives") + + // (2) A key the new store updated: the old binary serves the STALE file + // while the row says otherwise — a divergent pair. + stale := e.legacyGet(updated) + require.Equal(t, "2", stale.ResourceVersion, "the file is the pre-flip version") + require.Equal(t, "3", e.inspect(updated).jsonRV, "the row is the post-flip version") + + // (3) The old binary's row write, exactly as its writeMetadata issues it, + // nulls rv/uid: the CAS can never match again. + other := e.key("plain-01") + _, _, kind, _, ns, name := K8sPathToKeys(other) + require.NoError(t, sqlitex.Execute(e.fixture, + `INSERT OR REPLACE INTO metadata (kind, namespace, name, metadata) SELECT kind, namespace, name, metadata FROM metadata WHERE kind = ? AND namespace = ? AND name = ?`, + &sqlitex.ExecOptions{Args: []any{kind, ns, name}})) + row = e.inspect(other) + require.Nil(t, row.rv, "INSERT OR REPLACE nulled rv") + require.Nil(t, row.uid, "INSERT OR REPLACE nulled uid") + e.startNew() + shortCtx, cancel := context.WithTimeout(e.ctx, 700*time.Millisecond) + err = e.storeUpdate(shortCtx, other) + cancel() + require.Error(t, err, "rv NULL never matches the CAS") +} + +// storeGetLegacy is the old binary's Get. +func (e *migrationEnv) storeGetLegacy(key string) (*softwarecomposition.ContainerProfile, error) { + out := &softwarecomposition.ContainerProfile{} + err := e.legacy.Get(e.ctx, key, storage.GetOptions{}, out) + return out, err +} + +// The round trip: legacy → migrate → export → the old binary serves every +// object exactly as it did before the migration, at the same RV and UID, +// and can keep writing it. +// +// Bytes are compared as decoded objects, not raw: gob encodes maps +// (labels, annotations) in Go's randomised map order, so two encodes of the +// same object differ byte-wise; and the JSON codec keeps creationTimestamp +// to the second and empty collections as empty (canonicalCP), both +// documented codec deltas of the backend. Everything the old binary reads +// back is asserted equal. +func TestExport_RoundTripIsBehaviourallyIdentical(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + before := e.seed(4) + original := map[string][]byte{} + for k := range before { + original[k] = e.readFile(filepath.Base(k)) + } + + e.startNew() + report := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, len(before), report.Count(MigrationShapeMigrated)) + exported := e.export(ContainerProfileExportOptions{BatchSize: 3}) + require.Equal(t, len(before), exported.Exported) + require.Equal(t, 0, exported.LegacySkipped) + require.Equal(t, 0, exported.Undecodable) + e.stopNew() + + for k, legacyBefore := range before { + name := filepath.Base(k) + fileNow := e.readFile(name) + require.Equal(t, canonicalCP(decodeGob(t, original[k])), canonicalCP(decodeGob(t, fileNow)), "the exported file decodes to the original for %s", k) + got := e.legacyGet(k) + require.Equal(t, canonicalCP(legacyBefore), canonicalCP(got), "the old binary serves the pre-migration object for %s", k) + require.Equal(t, legacyBefore.ResourceVersion, got.ResourceVersion) + require.Equal(t, legacyBefore.UID, got.UID) + staged, err := afero.Exists(e.fs, e.filePath(name)+".t") + require.NoError(t, err) + require.False(t, staged, "no staging file left behind") + } + // The old binary keeps working on the exported files. + k := e.key("plain-01") + after := e.legacyUpdate(k, func(cp *softwarecomposition.ContainerProfile) { cp.Annotations["export-test"] = "old-binary-write" }) + require.Equal(t, "2", after.ResourceVersion) + require.Equal(t, "old-binary-write", e.legacyGet(k).Annotations["export-test"]) +} + +// The full rollback cycle in the design's order: export, downgrade, the old +// binary reads and writes, re-enable — the reconcile repairs what the old +// binary rewrote and nothing is lost. +func TestExport_ThenDowngradeThenReEnable(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + e.seed(2) + updated, created, untouched := e.key("plain-00"), e.key("created-under-the-new-store"), e.key("plain-01") + e.startNew() + e.mustMigrate(ContainerProfileMigrationOptions{}) + e.storeCreate("created-under-the-new-store") + require.NoError(t, e.storeUpdate(e.ctx, updated)) + newView := map[string]*softwarecomposition.ContainerProfile{ + updated: e.mustStoreGet(updated), created: e.mustStoreGet(created), untouched: e.mustStoreGet(untouched), + } + + // Export, then downgrade. + report := e.export(ContainerProfileExportOptions{}) + require.Equal(t, 5, report.Exported, "2 plain + 2 TS seeded, 1 created") + e.stopNew() + + // The old binary sees every object at its post-flip state. + for k, want := range newView { + got := e.legacyGet(k) + require.Equal(t, canonicalCP(want), canonicalCP(got), "%s", k) + require.Equal(t, want.ResourceVersion, got.ResourceVersion) + require.Equal(t, want.UID, got.UID) + } + require.True(t, e.inspect(created).metaExists, "nothing was deleted") + + // The old binary writes one key and deletes another. + e.legacyUpdate(created, func(cp *softwarecomposition.ContainerProfile) { cp.Annotations["export-test"] = "old-binary-write" }) + require.Nil(t, e.inspect(created).rv) + require.NoError(t, e.legacy.Delete(e.ctx, untouched, &softwarecomposition.ContainerProfile{}, nil, nil, nil, storage.DeleteOptions{})) + require.False(t, e.inspect(untouched).payloadExists, "Part 1: the old binary's delete now cleans up the payloads row too") + + // Re-enable: the every-start reconcile repairs both shapes. + e.startNew() + again := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 1, again.Counts[MigrationShapeLegacyRewrite+"/"+MigrationSourceFile], "%v", again.Counts) + require.Equal(t, 0, again.Count(MigrationShapeOrphanPayload), "%v", again.Counts) + require.Equal(t, 0, again.Count(MigrationShapeRowWithoutFile)) + e.assertINV2(updated, created, untouched) + got := e.mustStoreGet(created) + require.Equal(t, "old-binary-write", got.Annotations["export-test"], "the old binary's write survives the re-enable") + require.Equal(t, "2", got.ResourceVersion) + require.Equal(t, canonicalCP(newView[updated]), canonicalCP(e.mustStoreGet(updated)), "an untouched key is exactly as the new store left it") + _, err := e.storeGet(untouched) + require.True(t, storage.IsNotFound(err), "the old binary's delete holds") + require.NoError(t, e.storeUpdate(e.ctx, created), "the CAS is live again") + require.NoError(t, e.storeUpdate(e.ctx, updated)) +} + +func TestExport_SkipsLegacyRowsAndDryRunWritesNothing(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + e.seed(1) + migratedKey := e.key("plain-00") + e.startNew() + e.mustMigrate(ContainerProfileMigrationOptions{}) + e.storeCreate("created-under-the-new-store") + // A row a legacy writer already rewrote (rv NULL): its file is the legacy + // writer's, and the export must not overwrite it with the stale body. + e.stopNew() + e.legacyUpdate(migratedKey, func(cp *softwarecomposition.ContainerProfile) { cp.Annotations["export-test"] = "legacy" }) + legacyFile := e.readFile("plain-00") + + dry := e.export(ContainerProfileExportOptions{DryRun: true}) + require.Equal(t, 3, dry.Exported, "1 plain + 2 TS seeded, 1 created, 1 legacy-rewritten") + require.Equal(t, 1, dry.LegacySkipped) + exists, err := afero.Exists(e.fs, e.filePath("created-under-the-new-store")) + require.NoError(t, err) + require.False(t, exists, "dry-run wrote nothing") + + real := e.export(ContainerProfileExportOptions{}) + require.Equal(t, 3, real.Exported) + require.Equal(t, 1, real.LegacySkipped) + require.Equal(t, legacyFile, e.readFile("plain-00"), "the legacy writer's file is untouched") + + // An rv NULL row with NO file (no known producer): the body is the only + // copy, exported at the JSON's resourceVersion rather than skipped. + require.NoError(t, e.fs.Remove(e.filePath("plain-00"))) + filled := e.export(ContainerProfileExportOptions{}) + require.Equal(t, 4, filled.Exported) + require.Equal(t, 0, filled.LegacySkipped) + filledObj := e.legacyGet(migratedKey) + require.Equal(t, "3", filledObj.ResourceVersion, "the row JSON's version (seed updated it to 2, the legacy write to 3)") + require.Empty(t, filledObj.Annotations["export-test"], "the payloads body (pre-legacy-write) is what was left") + got := e.legacyGet(e.key("created-under-the-new-store")) + require.Equal(t, "1", got.ResourceVersion) + require.Equal(t, helpersv1.Learning, got.Annotations[helpersv1.StatusMetadataKey]) +} + +// TestExport_RemovesStaleFileOfKeyDeletedUnderTheNewStore reproduces the +// rollback-resurrection sequence: migrate A, delete A under the ObjectStore +// backend (which removes only the database rows and leaves A's pre-flip .g +// file on disk), export, downgrade. Without reconcileStaleExportedFiles, the +// old binary's get() finds the stale file and A is readable again after +// being deleted. +func TestExport_RemovesStaleFileOfKeyDeletedUnderTheNewStore(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + e.seed(1) + key := e.key("plain-00") + e.startNew() + e.mustMigrate(ContainerProfileMigrationOptions{}) + + out := &softwarecomposition.ContainerProfile{} + require.NoError(t, e.store.Delete(e.ctx, key, out, nil, nil, nil, storage.DeleteOptions{})) + row := e.inspect(key) + require.False(t, row.metaExists, "the delete removed the metadata row") + require.False(t, row.payloadExists, "and the payloads row") + exists, err := afero.Exists(e.fs, e.filePath("plain-00")) + require.NoError(t, err) + require.True(t, exists, "the pre-flip legacy file survives an ObjectStore delete") + + report := e.export(ContainerProfileExportOptions{}) + require.Equal(t, 1, report.StaleFilesRemoved) + exists, err = afero.Exists(e.fs, e.filePath("plain-00")) + require.NoError(t, err) + require.False(t, exists, "the export removed the stale file") + + e.stopNew() + getOut := &softwarecomposition.ContainerProfile{} + err = e.legacy.Get(e.ctx, key, storage.GetOptions{}, getOut) + require.True(t, storage.IsNotFound(err), "the old binary must not resurrect the deleted object") +} + +// TestExport_DryRunLeavesStaleFiles confirms the dry-run count matches what +// a real run would remove, without touching the filesystem. +func TestExport_DryRunLeavesStaleFiles(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + e.seed(1) + key := e.key("plain-00") + e.startNew() + e.mustMigrate(ContainerProfileMigrationOptions{}) + out := &softwarecomposition.ContainerProfile{} + require.NoError(t, e.store.Delete(e.ctx, key, out, nil, nil, nil, storage.DeleteOptions{})) + + dry := e.export(ContainerProfileExportOptions{DryRun: true}) + require.Equal(t, 1, dry.StaleFilesRemoved) + exists, err := afero.Exists(e.fs, e.filePath("plain-00")) + require.NoError(t, err) + require.True(t, exists, "dry-run removed nothing") +} + +// TestExport_TrailingSlashRootDoesNotDeleteLiveFiles: reconcileStaleExportedFiles +// derives a key by slicing a Walk()-reported path (always cleaned, via +// filepath.Join) at len(root). An uncleaned root with a trailing slash (e.g. +// the CLI's -root /data/) makes that length one too many, silently dropping +// the key's required leading '/' -- ReadMetadata then misses the live row +// for a key that was NEVER deleted, and the stale-file pass wrongly deletes +// its just-exported file. Both a live object and a genuinely deleted one are +// present so the fix is proven both ways: the live file must survive, the +// deleted one's stale file must still be removed. +func TestExport_TrailingSlashRootDoesNotDeleteLiveFiles(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + e.seed(2) + liveKey, deletedKey := e.key("plain-00"), e.key("plain-01") + e.startNew() + e.mustMigrate(ContainerProfileMigrationOptions{}) + + out := &softwarecomposition.ContainerProfile{} + require.NoError(t, e.store.Delete(e.ctx, deletedKey, out, nil, nil, nil, storage.DeleteOptions{})) + liveExists, err := afero.Exists(e.fs, e.filePath("plain-00")) + require.NoError(t, err) + require.True(t, liveExists, "plain-00's row is live, its pre-flip file still on disk") + + report, err := ExportContainerProfiles(e.ctx, e.pool, e.fs, DefaultStorageRoot+"/", e.scheme, ContainerProfileExportOptions{}) + require.NoError(t, err) + require.Equal(t, 1, report.StaleFilesRemoved, "only plain-01's stale file, not plain-00's live one") + + liveExists, err = afero.Exists(e.fs, e.filePath("plain-00")) + require.NoError(t, err) + require.True(t, liveExists, "the trailing slash must not make the live file look stale") + + deletedExists, err := afero.Exists(e.fs, e.filePath("plain-01")) + require.NoError(t, err) + require.False(t, deletedExists, "the genuinely deleted key's stale file is still removed") + + e.stopNew() + getOut := &softwarecomposition.ContainerProfile{} + require.NoError(t, e.legacy.Get(e.ctx, liveKey, storage.GetOptions{}, getOut), "the live object must still be readable by an old binary") +} + +// failOpenFs fails Open for one target path while armed, otherwise +// delegates -- reproduces a permission-denied/I/O error reading a specific +// file's content (what decodeLegacyFileAt hits), distinct from toggleFailFs +// (migration test helper, same package) which fails Stat for directory/walk +// -level errors. afero.Walk itself never calls Open, only Stat/ReadDir, so +// arming this on a file already discovered by Walk reproduces the failure +// happening exactly where reconcileStaleExportedFiles decodes it. +type failOpenFs struct { + afero.Fs + target string + fail atomic.Bool +} + +func (f *failOpenFs) Open(name string) (afero.File, error) { + if f.fail.Load() && name == f.target { + return nil, fmt.Errorf("failOpenFs: simulated open failure for %s", name) + } + return f.Fs.Open(name) +} + +// failReadFs succeeds Open for one target path while armed, but returns a +// file whose Read always errors -- reproduces a device/mount that opens +// fine but errors mid-read (a bad sector, a network filesystem hiccup), +// distinct from failOpenFs (Open itself fails). decodeLegacyFileAt must +// treat this exactly like an Open failure (errLegacyFileAccess), not let +// gob's Decode fold it into an indistinguishable "content is malformed" +// error. +type failReadFs struct { + afero.Fs + target string + fail atomic.Bool +} + +func (f *failReadFs) Open(name string) (afero.File, error) { + file, err := f.Fs.Open(name) + if err != nil || !f.fail.Load() || name != f.target { + return file, err + } + return &failReadFile{File: file}, nil +} + +type failReadFile struct{ afero.File } + +func (f *failReadFile) Read(_ []byte) (int, error) { + return 0, fmt.Errorf("failReadFile: simulated read failure for %s", f.Name()) +} + +// TestExport_ReconcileTopLevelStatErrorFailsExport: the same class of bug as +// TestMigration_SweepTopLevelStatErrorDoesNotMarkDone, on the export side -- +// afero.DirExists returns exists=false on ANY stat error, not just "does +// not exist", and reconcileStaleExportedFiles used to read that as "nothing +// to reconcile" and report success. A stat error there must instead fail +// the export, since a directory that could not be checked might hold a +// stale, resurrection-capable file the export never got to look at. +func TestExport_ReconcileTopLevelStatErrorFailsExport(t *testing.T) { + failing := &toggleFailFs{Fs: afero.NewMemMapFs()} + e := newMigrationEnv(t, failing) + e.seed(1) + e.startNew() + e.mustMigrate(ContainerProfileMigrationOptions{}) + + failing.target = filepath.Join(DefaultStorageRoot, softwarecomposition.GroupName, ContainerProfileKind) + failing.fail.Store(true) + _, err := ExportContainerProfiles(e.ctx, e.pool, failing, DefaultStorageRoot, e.scheme, ContainerProfileExportOptions{}) + require.Error(t, err, "a stat error checking the reconcile directory must fail the export, not read as \"nothing to reconcile\"") +} + +// TestExport_ReconcileFileAccessErrorFailsExport: a permission/I/O error +// opening a stale candidate's file (its metadata row is gone, its file +// still exists) used to be treated identically to the file's content being +// genuinely undecodable garbage -- reconcileStaleExportedFiles left it in +// place and the export still reported success. But an access error means +// the tool could not tell whether that file is content an old binary would +// resurrect; unlike truly undecodable content (safe to leave, an old binary +// can't read it either), it must fail the export instead of silently +// leaving a landmine an operator believes was checked. +func TestExport_ReconcileFileAccessErrorFailsExport(t *testing.T) { + failing := &failOpenFs{Fs: afero.NewMemMapFs()} + e := newMigrationEnv(t, failing) + e.seed(1) + key := e.key("plain-00") + e.startNew() + e.mustMigrate(ContainerProfileMigrationOptions{}) + + out := &softwarecomposition.ContainerProfile{} + require.NoError(t, e.store.Delete(e.ctx, key, out, nil, nil, nil, storage.DeleteOptions{})) + exists, err := afero.Exists(e.fs, e.filePath("plain-00")) + require.NoError(t, err) + require.True(t, exists, "the pre-flip legacy file survives an ObjectStore delete") + + failing.target = e.filePath("plain-00") + failing.fail.Store(true) + _, err = ExportContainerProfiles(e.ctx, e.pool, failing, DefaultStorageRoot, e.scheme, ContainerProfileExportOptions{}) + require.Error(t, err, "an access error on a stale candidate must fail the export, not be treated as harmless undecodable content") + + failing.fail.Store(false) + exists, err = afero.Exists(e.fs, e.filePath("plain-00")) + require.NoError(t, err) + require.True(t, exists, "the file was never classified, so it must not have been removed either") +} + +// TestExport_ReconcileOpenSucceedsReadFailsFailsExport: a stale candidate's +// file can open successfully and still fail to be read (a bad sector, a +// device/mount that opens fine but errors mid-read) -- errLegacyFileAccess +// used to wrap only fs.Open's own error, so a Read failure surfaced as an +// opaque, untagged error from gob's Decode, indistinguishable from the +// file's content genuinely being malformed. reconcileStaleExportedFiles +// then left it in place and the export still reported success; after +// downgrade, an old binary reading that now-otherwise-fine file could +// resurrect a deleted object. +func TestExport_ReconcileOpenSucceedsReadFailsFailsExport(t *testing.T) { + failing := &failReadFs{Fs: afero.NewMemMapFs()} + e := newMigrationEnv(t, failing) + e.seed(1) + key := e.key("plain-00") + e.startNew() + e.mustMigrate(ContainerProfileMigrationOptions{}) + + out := &softwarecomposition.ContainerProfile{} + require.NoError(t, e.store.Delete(e.ctx, key, out, nil, nil, nil, storage.DeleteOptions{})) + exists, err := afero.Exists(e.fs, e.filePath("plain-00")) + require.NoError(t, err) + require.True(t, exists, "the pre-flip legacy file survives an ObjectStore delete") + + failing.target = e.filePath("plain-00") + failing.fail.Store(true) + _, err = ExportContainerProfiles(e.ctx, e.pool, failing, DefaultStorageRoot, e.scheme, ContainerProfileExportOptions{}) + require.Error(t, err, "Open succeeding then Read failing must fail the export, not be silently folded into \"undecodable content\"") + + failing.fail.Store(false) + exists, err = afero.Exists(e.fs, e.filePath("plain-00")) + require.NoError(t, err) + require.True(t, exists, "the file was never classified, so it must not have been removed either") +} diff --git a/pkg/registry/file/sqliteobject_gate.go b/pkg/registry/file/sqliteobject_gate.go new file mode 100644 index 000000000..1f1f6b430 --- /dev/null +++ b/pkg/registry/file/sqliteobject_gate.go @@ -0,0 +1,467 @@ +package file + +// The process's write gate: a caller-side two-lane FIFO ticket semaphore +// owning ONE dedicated write connection (design: +// .omc/plans/full-acid-storage-architecture.md §6.4, shared with the legacy +// kinds per .omc/plans/write-gate-sharing.md §3). +// +// Acquisition order, fixed: prepare (pool connection for reads, released) → +// gate ticket (queued on ctx) → BEGIN IMMEDIATE on the gate's connection → +// statements → COMMIT → release ticket → dispatch. No pool connection is held +// while queued on the gate on the hot paths, and no ticket is held while +// waiting on the pool. +// +// Not a sync.Mutex: Go's mutex is not FIFO and cannot be abandoned on ctx +// cancellation. The releaser hands the ticket directly to the next waiter +// (high lane first, a waiting low job forced through after highBurstLimit +// consecutive high commits — the policy of singleWriter.run), so fairness is +// FIFO among gated writers. A waiter whose ctx fires after the ticket was +// granted hands the ticket back (INV-5): a leaked ticket would wedge every +// writer forever. +// +// One gate per pool (writegate_registry.go): after R1 the gate is the only +// code that acquires SQLite's write lock; any other writer busy-waits against +// it for the whole busy timeout, invisible to the gate's own histograms. + +import ( + "context" + "errors" + "fmt" + "runtime" + "runtime/debug" + "sync" + "sync/atomic" + "time" + + "github.com/kubescape/go-logger" + "github.com/kubescape/go-logger/helpers" + "github.com/kubescape/storage/pkg/metrics" + "k8s.io/apiserver/pkg/storage" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitemigration" + "zombiezen.com/go/sqlite/sqlitex" +) + +// errGateClosed is returned to writers queued on, or arriving at, a gate whose +// Close has begun. +var errGateClosed = errors.New("write gate: closed") + +// errGateReentrant is returned when a gated fn tries to acquire the gate again +// through the ctx it was handed: the gate is not re-entrant, and the nested +// acquire would queue behind its own holder forever (INV-1′). In tests the +// same condition panics, so a site that swallows the error (`_ = +// DeleteMetadata(...)`) still fails loudly. +var errGateReentrant = errors.New("write gate: re-entrant acquire from inside a gated transaction") + +// gateReentrantPanics makes a re-entrant acquire panic instead of returning +// errGateReentrant. Set by the package's TestMain; false in production. +var gateReentrantPanics atomic.Bool + +// gateCtxKey marks the ctx a gated fn receives; valid only for the dynamic +// extent of fn (R-5): a site must not store it in anything that outlives the +// call, or a legitimate later write through the stored ctx is refused. +type gateCtxKey struct{} + +// ErrWriteConflict is the exported alias of the single writer's conflict +// sentinel: an ObjectStore compare-and-swap (UPDATE/DELETE … WHERE rv=:rv AND +// uid=:uid) matched no row because another write committed first. Callers +// re-read and retry; the consolidation pass retries once. +var ErrWriteConflict = errWriteConflict + +// Watchdog tunables (PM-G2). Package-level vars so tests can shrink them. +var ( + // gateWatchdogInterval is how often the watchdog samples the hold. + gateWatchdogInterval = time.Second + // gateWatchdogThreshold is the hold age past which the watchdog logs the + // holder once, with every goroutine's stack: a leaked ticket, a re-entrant + // acquire the ctx marker did not see, or a holder blocked on a PV that + // stopped responding. + gateWatchdogThreshold = 60 * time.Second +) + +// WriteGate is the exported name of the process's write gate for main.go's +// wiring; everything else in this package uses writeGate. +type WriteGate = writeGate + +// NewWriteGate builds the process's one write gate over pool. It is created +// beside the pool when config.ContainerProfileSqliteBackend is on and handed +// to the ObjectStore, the legacy StorageImpl and the cleanup handler; with the +// flag off no gate exists and every legacy write site runs today's code. +func NewWriteGate(ctx context.Context, pool *sqlitemigration.Pool) (*WriteGate, error) { + return newWriteGate(ctx, pool) +} + +type gateWaiter struct { + ch chan struct{} + priority writePriority + // granted is set under writeGate.mu at the instant the ticket is handed + // over; a cancelled waiter that finds it set owns the gate and must release. + granted bool + // rejected is set under writeGate.mu when Close drains the queue. + rejected bool +} + +type writeGate struct { + pool *sqlitemigration.Pool + + mu sync.Mutex + idle *sync.Cond + busy bool + high, low []*gateWaiter + highStreak int + closed bool + // holdStart/holdPath/holdKind describe the current holder (busy) for the + // watchdog and the hold-age gauge. + holdStart time.Time + holdPath, holdKind string + // conn is the dedicated write connection, taken once from the pool and + // returned only by Close (K-5: sqlitex.Pool.Close blocks until every + // connection is back). + conn *sqlite.Conn + // owned is every connection this gate has EVER held — the initial one and + // each replacement, never removed — mapped to the write-statement sequence + // at which the gate took it. A write recorded on a connection the gate + // held at the time is a gated write even after cleanOrReplace swapped it + // out (AC-G1, CR-1); a write recorded on it BEFORE the gate took it (a pool + // taker's, when it was still a pool connection) is not. A closed pointer + // stays reachable from the records, so the runtime cannot reuse it. + // Bounded at pool size + 1: each replacement consumes one pool connection + // for good. + owned map[*sqlite.Conn]uint64 + // closeSeq is the write-statement sequence recorded by Close. Close puts + // the dedicated connection back in the pool, so a later non-gate taker + // could write on an owned pointer: only records older than closeSeq are + // the gate's (R-8). + closeSeq uint64 + // replaced counts connections that could not be cleaned after a failed + // transaction and were replaced from the pool. + replaced int + + watchdogStop chan struct{} + watchdogDone chan struct{} + watchdogFired atomic.Int64 +} + +// newWriteGate takes the gate's dedicated connection from pool and registers +// the gate as the pool's one writer; a pool that already has a live gate is +// refused (errGateExists) — two gates on one pool would busy-wait against +// each other while each reports every write as gated (PM-G8). +func newWriteGate(ctx context.Context, pool *sqlitemigration.Pool) (*writeGate, error) { + conn, err := pool.Take(ctx) + if err != nil { + return nil, fmt.Errorf("write gate: take dedicated connection: %w", err) + } + // Pool.Take bound the interrupt to ctx; the connection outlives it. + conn.SetInterrupt(nil) + g := &writeGate{ + pool: pool, + conn: conn, + owned: map[*sqlite.Conn]uint64{conn: writeStmtSeq.Load()}, + watchdogStop: make(chan struct{}), + watchdogDone: make(chan struct{}), + } + g.idle = sync.NewCond(&g.mu) + if err := registerWriteGate(pool, g); err != nil { + pool.Put(conn) + return nil, err + } + if obs := writeGateObserver.Load(); obs != nil { + (*obs)(g) + } + go g.watchdog() + return g, nil +} + +// owns reports whether a write statement recorded as seq on conn was the +// gate's own: conn is one the gate has ever held, and the record falls +// inside the gate's tenure of it — after the gate took it and before Close +// (a live gate has closeSeq 0). +func (g *writeGate) owns(conn *sqlite.Conn, seq uint64) bool { + g.mu.Lock() + defer g.mu.Unlock() + openSeq, ok := g.owned[conn] + if !ok || seq <= openSeq { + return false + } + return g.closeSeq == 0 || seq < g.closeSeq +} + +// acquire blocks until the caller owns the gate or ctx is done. On a nil +// return the caller MUST call release. +func (g *writeGate) acquire(ctx context.Context, priority writePriority) error { + g.mu.Lock() + if g.closed { + g.mu.Unlock() + return errGateClosed + } + if !g.busy { + g.busy = true + g.mu.Unlock() + metrics.ObserveWriteGateWait(priority.label(), 0) + return nil + } + w := &gateWaiter{ch: make(chan struct{}, 1), priority: priority} + if priority == priorityHigh { + g.high = append(g.high, w) + } else { + g.low = append(g.low, w) + } + g.mu.Unlock() + + start := time.Now() + select { + case <-w.ch: + metrics.ObserveWriteGateWait(priority.label(), time.Since(start)) + if w.rejected { + return errGateClosed + } + return nil + case <-ctx.Done(): + g.mu.Lock() + if w.granted { + // The ticket arrived in the same instant: we own the gate. Hand it + // back instead of leaking it (INV-5). + g.mu.Unlock() + <-w.ch + g.release() + return ctx.Err() + } + g.removeWaiter(w) + g.mu.Unlock() + return ctx.Err() + } +} + +// removeWaiter removes w from its lane; caller holds g.mu. +func (g *writeGate) removeWaiter(w *gateWaiter) { + lane := &g.low + if w.priority == priorityHigh { + lane = &g.high + } + for i, x := range *lane { + if x == w { + *lane = append((*lane)[:i], (*lane)[i+1:]...) + return + } + } +} + +// pickNext implements the two-lane policy of singleWriter.run: high first, +// unless highBurstLimit consecutive high commits have run while a low job +// waited, in which case the low job goes through. Caller holds g.mu. +func (g *writeGate) pickNext() *gateWaiter { + if g.highStreak < highBurstLimit && len(g.high) > 0 { + w := g.high[0] + g.high = g.high[1:] + g.highStreak++ + return w + } + if len(g.low) > 0 { + w := g.low[0] + g.low = g.low[1:] + g.highStreak = 0 + return w + } + if len(g.high) > 0 { + w := g.high[0] + g.high = g.high[1:] + g.highStreak++ + return w + } + return nil +} + +// release hands the gate to the next queued writer, or marks it idle. +func (g *writeGate) release() { + g.mu.Lock() + defer g.mu.Unlock() + g.holdStart, g.holdPath, g.holdKind = time.Time{}, "", "" + if next := g.pickNext(); next != nil { + next.granted = true + next.ch <- struct{}{} + return + } + g.busy = false + g.idle.Broadcast() +} + +// queued reports how many writers are waiting, for tests and gauges. +func (g *writeGate) queued() (high, low int) { + g.mu.Lock() + defer g.mu.Unlock() + return len(g.high), len(g.low) +} + +// held reports whether some writer currently owns the gate. +func (g *writeGate) held() bool { + g.mu.Lock() + defer g.mu.Unlock() + return g.busy +} + +// run executes fn inside BEGIN IMMEDIATE … COMMIT on the gate's connection, +// holding the gate for exactly that span. fn must contain only SQL on +// already-prepared bytes (INV-1; the legacy kinds' INV-1′ adds one rename of +// a file this process finished writing before the ticket). fn receives a +// ctx marked as gate-held: a nested acquire through it fails at O(1) +// (errGateReentrant) instead of queuing behind its own holder. A non-nil +// error from fn rolls the transaction back and is returned; a panic in fn is +// recovered, rolled back, counted under CommitOutcomePanic and returned as +// an error (L0-B). path labels the hold, kind the commit outcome. +func (g *writeGate) run(ctx context.Context, priority writePriority, path, kind string, fn func(ctx context.Context, conn *sqlite.Conn) error) (err error) { + if held := ctx.Value(gateCtxKey{}); held != nil { + metrics.IncWriteGateReentrant(path) + if gateReentrantPanics.Load() { + panic(fmt.Sprintf("%v: %s inside %s", errGateReentrant, path, held)) + } + return fmt.Errorf("%w: %s inside %s", errGateReentrant, path, held) + } + if err := g.acquire(ctx, priority); err != nil { + return err + } + defer g.release() + g.mu.Lock() + g.holdStart, g.holdPath, g.holdKind = time.Now(), path, kind + conn := g.conn + g.mu.Unlock() + + start := time.Now() + defer func() { metrics.ObserveSqliteWriteHold(path, kind, time.Since(start)) }() + + defer func() { + if r := recover(); r != nil { + logger.L().Error("write gate: panic inside the gated transaction, rolled back", + helpers.String("path", path), helpers.String("kind", kind), helpers.Interface("panic", r), helpers.String("stack", string(debug.Stack()))) + metrics.IncSingleWriterCommit(kind, priority.label(), metrics.CommitOutcomePanic) + g.cleanOrReplace(conn) + if perr, ok := r.(error); ok { + err = fmt.Errorf("write gate: panic during %s transaction: %w", path, perr) + } else { + err = fmt.Errorf("write gate: panic during %s transaction: %v", path, r) + } + } + }() + + beforeBegin := time.Now() + endFn, err := sqlitex.ImmediateTransaction(conn) + metrics.ObserveSqliteBusyWait(time.Since(beforeBegin)) + if err != nil { + g.cleanOrReplace(conn) + return fmt.Errorf("BEGIN IMMEDIATE: %w", err) + } + err = fn(context.WithValue(ctx, gateCtxKey{}, path), conn) + endFn(&err) + if err != nil { + g.cleanOrReplace(conn) + if errors.Is(err, errWriteConflict) || storage.IsExist(err) { + metrics.IncSingleWriterCommit(kind, priority.label(), metrics.CommitOutcomeConflict) + } else { + metrics.IncSingleWriterCommit(kind, priority.label(), metrics.CommitOutcomeError) + } + return err + } + metrics.IncSingleWriterCommit(kind, priority.label(), metrics.CommitOutcomeCommitted) + return nil +} + +// cleanOrReplace makes sure the gate's connection is out of any transaction +// after a failure. A connection that cannot be cleaned is replaced from the +// pool, never dropped: the gate must never end up without a connection. +// Caller owns the gate. +func (g *writeGate) cleanOrReplace(conn *sqlite.Conn) { + if conn.AutocommitEnabled() { + return + } + _ = sqlitex.ExecuteTransient(conn, "ROLLBACK", nil) + if conn.AutocommitEnabled() { + return + } + logger.L().Error("write gate: connection could not be cleaned, replacing it from the pool") + ctx, cancel := context.WithTimeout(context.Background(), poolTimeout) + defer cancel() + fresh, err := g.pool.Take(ctx) + if err != nil { + logger.L().Error("write gate: could not take a replacement connection; keeping the dirty one", helpers.Error(err)) + return + } + fresh.SetInterrupt(nil) + openSeq := writeStmtSeq.Load() + // The dirty connection is closed, not returned: a mid-transaction + // connection in the pool would poison whoever takes it next. This leaves + // the pool one short for Pool.Close (the K-5 note on Put(nil)). + _ = conn.Close() + g.mu.Lock() + g.conn = fresh + g.owned[fresh] = openSeq + g.replaced++ + g.mu.Unlock() +} + +// watchdog samples the hold every gateWatchdogInterval: it keeps the +// hold-age gauge current and, once per hold, logs the holder with every +// goroutine's stack when the hold outlives gateWatchdogThreshold (PM-G2). +func (g *writeGate) watchdog() { + defer close(g.watchdogDone) + ticker := time.NewTicker(gateWatchdogInterval) + defer ticker.Stop() + logged := false + for { + select { + case <-g.watchdogStop: + metrics.SetWriteGateHoldAge(0) + return + case <-ticker.C: + } + g.mu.Lock() + busy, start, path, kind := g.busy, g.holdStart, g.holdPath, g.holdKind + g.mu.Unlock() + if !busy || start.IsZero() { + metrics.SetWriteGateHoldAge(0) + logged = false + continue + } + age := time.Since(start) + metrics.SetWriteGateHoldAge(age) + if age > gateWatchdogThreshold && !logged { + logged = true + g.watchdogFired.Add(1) + buf := make([]byte, 1<<20) + n := runtime.Stack(buf, true) + logger.L().Error("write gate: held past the watchdog threshold; every writer of every kind is queued behind it", + helpers.String("path", path), helpers.String("kind", kind), helpers.String("age", age.String()), helpers.String("stacks", string(buf[:n]))) + } + } +} + +// Close stops admitting writers, waits for the current holder to finish, and +// returns the dedicated connection to the pool so that Pool.Close can +// complete (K-5). Idempotent. +func (g *writeGate) Close() error { + g.mu.Lock() + if g.closed { + g.mu.Unlock() + return nil + } + g.closed = true + for _, w := range append(g.high, g.low...) { + w.rejected = true + w.ch <- struct{}{} + } + g.high, g.low = nil, nil + for g.busy { + g.idle.Wait() + } + conn := g.conn + g.conn = nil + // Every gated write was recorded before this point; anything on the + // returned connection from here on is somebody else's (R-8). + g.closeSeq = writeStmtSeq.Add(1) + g.mu.Unlock() + close(g.watchdogStop) + <-g.watchdogDone + unregisterWriteGate(g.pool, g) + if conn != nil { + g.pool.Put(conn) + } + return nil +} diff --git a/pkg/registry/file/sqliteobject_gate_test.go b/pkg/registry/file/sqliteobject_gate_test.go new file mode 100644 index 000000000..47e3505ec --- /dev/null +++ b/pkg/registry/file/sqliteobject_gate_test.go @@ -0,0 +1,312 @@ +package file + +import ( + "context" + "errors" + "math/rand" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitex" +) + +func newTestGate(t *testing.T, poolSize int) (*writeGate, func()) { + t.Helper() + pool := NewPoolWithOptions(t.TempDir()+"/gate.sq3", PoolOptions{Size: poolSize, DisableAutoCheckpoint: true}) + armUngatedWriteCheck(t, pool) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + g, err := newWriteGate(ctx, pool) + require.NoError(t, err) + return g, func() { + require.NoError(t, g.Close()) + done := make(chan error, 1) + go func() { done <- pool.Close() }() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(10 * time.Second): + t.Fatal("pool.Close hung: the gate did not return its connection (K-5)") + } + } +} + +// TestWriteGate_FIFOWithinLane: the holder hands the ticket to waiters in +// enqueue order. +func TestWriteGate_FIFOWithinLane(t *testing.T) { + g, done := newTestGate(t, 2) + defer done() + require.NoError(t, g.acquire(context.Background(), priorityHigh)) + + const n = 8 + var order []int + var mu sync.Mutex + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + require.NoError(t, g.acquire(context.Background(), priorityHigh)) + mu.Lock() + order = append(order, i) + mu.Unlock() + g.release() + }(i) + // Enqueue deterministically: wait until this waiter is queued. + require.Eventually(t, func() bool { h, _ := g.queued(); return h == i+1 }, 5*time.Second, time.Millisecond) + } + g.release() + wg.Wait() + want := make([]int, n) + for i := range want { + want[i] = i + } + assert.Equal(t, want, order) +} + +// TestWriteGate_HighBurstLimitForcesLow: with a low job queued, at most +// highBurstLimit consecutive high jobs run before it (singleWriter.run's +// policy, re-hosted). +func TestWriteGate_HighBurstLimitForcesLow(t *testing.T) { + g, done := newTestGate(t, 2) + defer done() + require.NoError(t, g.acquire(context.Background(), priorityHigh)) + + var order []string + var mu sync.Mutex + var wg sync.WaitGroup + enqueue := func(label string, p writePriority, expectHigh, expectLow int) { + wg.Add(1) + go func() { + defer wg.Done() + require.NoError(t, g.acquire(context.Background(), p)) + mu.Lock() + order = append(order, label) + mu.Unlock() + g.release() + }() + require.Eventually(t, func() bool { h, l := g.queued(); return h == expectHigh && l == expectLow }, 5*time.Second, time.Millisecond) + } + enqueue("low", priorityLow, 0, 1) + for i := 0; i < highBurstLimit+5; i++ { + enqueue("high", priorityHigh, i+1, 1) + } + g.release() + wg.Wait() + lowAt := -1 + for i, l := range order { + if l == "low" { + lowAt = i + } + } + assert.Equal(t, highBurstLimit, lowAt, "the low job runs after exactly highBurstLimit high jobs") +} + +// TestWriteGate_INV5_NoTicketLeakUnderCancellation cancels callers at every +// point of the hand-off — before enqueue, while queued, at the instant of +// grant, and while holding — under a race-heavy schedule, and asserts the +// gate always drains to idle and a fresh writer acquires promptly. +func TestWriteGate_INV5_NoTicketLeakUnderCancellation(t *testing.T) { + g, done := newTestGate(t, 2) + defer done() + rng := rand.New(rand.NewSource(1)) + + for round := 0; round < 40; round++ { + var wg sync.WaitGroup + var granted atomic.Int64 + for i := 0; i < 24; i++ { + wg.Add(1) + // draw on the test goroutine: math/rand.Rand is not goroutine-safe + timeout := time.Duration(rng.Intn(600)) * time.Microsecond + hold := time.Duration(rng.Intn(50)) * time.Microsecond + p := priorityHigh + if i%3 == 0 { + p = priorityLow + } + go func() { + defer wg.Done() + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + if err := g.acquire(ctx, p); err != nil { + return + } + granted.Add(1) + // hold briefly; sometimes our own ctx has already fired + time.Sleep(hold) + g.release() + }() + } + // The instant-of-grant race: a holder releases while a waiter's ctx + // expires; the waiter must either run or hand the ticket back. + wg.Wait() + require.Eventually(t, func() bool { + h, l := g.queued() + return !g.held() && h == 0 && l == 0 + }, 5*time.Second, 100*time.Microsecond, "round %d: gate did not drain (in-flight ticket leaked)", round) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + require.NoError(t, g.acquire(ctx, priorityHigh), "round %d: fresh writer could not acquire", round) + cancel() + g.release() + } +} + +// TestWriteGate_GrantAtCancelInstantHandsBack pins the exact hand-off race: +// the waiter's ctx is already done when the holder grants it the ticket. +func TestWriteGate_GrantAtCancelInstantHandsBack(t *testing.T) { + g, done := newTestGate(t, 2) + defer done() + for i := 0; i < 500; i++ { + require.NoError(t, g.acquire(context.Background(), priorityHigh)) + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { errCh <- g.acquire(ctx, priorityHigh) }() + require.Eventually(t, func() bool { h, _ := g.queued(); return h == 1 }, 5*time.Second, 10*time.Microsecond) + // Cancel and release "simultaneously". + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); cancel() }() + go func() { defer wg.Done(); g.release() }() + wg.Wait() + err := <-errCh + if err == nil { + g.release() + } else { + assert.ErrorIs(t, err, context.Canceled) + } + require.Eventually(t, func() bool { return !g.held() }, 2*time.Second, 10*time.Microsecond, "iteration %d leaked the ticket", i) + } +} + +// TestWriteGate_CloseRejectsWaiters: Close stops admitting, wakes queued +// writers with errGateClosed, waits for the holder and returns the connection +// (the deferred pool.Close proves K-5). +func TestWriteGate_CloseRejectsWaiters(t *testing.T) { + g, done := newTestGate(t, 2) + require.NoError(t, g.acquire(context.Background(), priorityHigh)) + waiterErr := make(chan error, 1) + go func() { waiterErr <- g.acquire(context.Background(), priorityLow) }() + require.Eventually(t, func() bool { _, l := g.queued(); return l == 1 }, 5*time.Second, time.Millisecond) + + closed := make(chan struct{}) + go func() { _ = g.Close(); close(closed) }() + select { + case <-closed: + t.Fatal("Close returned while the gate was still held") + case <-time.After(50 * time.Millisecond): + } + assert.ErrorIs(t, <-waiterErr, errGateClosed) + g.release() + select { + case <-closed: + case <-time.After(5 * time.Second): + t.Fatal("Close did not return after the holder released") + } + assert.ErrorIs(t, g.acquire(context.Background(), priorityHigh), errGateClosed) + done() +} + +// TestWriteGate_RunPanicContainment: a panic inside the transaction body is +// recovered, rolled back, and leaves the connection clean for the next writer. +func TestWriteGate_RunPanicContainment(t *testing.T) { + g, done := newTestGate(t, 2) + defer done() + err := g.run(context.Background(), priorityHigh, "test", "test", func(_ context.Context, conn *sqlite.Conn) error { + require.NoError(t, sqlitex.ExecuteTransient(conn, `INSERT INTO metadata (kind,namespace,name,metadata) VALUES ('k','n','panic','{}')`, nil)) + panic("boom") + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "panic") + assert.True(t, g.conn.AutocommitEnabled(), "connection left inside a transaction") + assert.False(t, g.held()) + + var n int64 + require.NoError(t, g.run(context.Background(), priorityHigh, "test", "test", func(_ context.Context, conn *sqlite.Conn) error { + return sqlitex.ExecuteTransient(conn, `SELECT count(*) FROM metadata WHERE name='panic'`, &sqlitex.ExecOptions{ + ResultFunc: func(stmt *sqlite.Stmt) error { n = stmt.ColumnInt64(0); return nil }, + }) + })) + assert.Equal(t, int64(0), n, "the panicked transaction must have been rolled back") + + err = g.run(context.Background(), priorityHigh, "test", "test", func(context.Context, *sqlite.Conn) error { return errors.New("no") }) + assert.EqualError(t, err, "no") + assert.True(t, g.conn.AutocommitEnabled()) +} + +// TestWriteGate_SecondGateOnPoolRefused (R-7, PM-G8): a pool carries one +// live gate. Two gates on two dedicated connections would each gate their +// own writes while busy-waiting against each other — the class, visible from +// neither side — so the second construction fails, and a closed gate frees +// the pool for a new one. +func TestWriteGate_SecondGateOnPoolRefused(t *testing.T) { + g, done := newTestGate(t, 3) + defer done() + ctx := context.Background() + second, err := newWriteGate(ctx, g.pool) + require.ErrorIs(t, err, errGateExists) + require.Nil(t, second) + require.NoError(t, g.Close()) + replacement, err := newWriteGate(ctx, g.pool) + require.NoError(t, err, "a closed gate frees the pool") + require.NoError(t, replacement.Close()) +} + +// TestWriteGate_SwapKeepsPreSwapConnectionOwned (T-G2, CR-1/CR-5): a gated +// write recorded on the gate's connection before cleanOrReplace swapped it +// out is still the gate's write. The swap is forced explicitly: fn runs a +// recorded INSERT, installs a closed interrupt channel on the connection and +// panics — the gate's own recovery path calls cleanOrReplace directly, whose +// ROLLBACK is then interrupted (the sqlitex end function is not on that path; +// it clears the interrupt before its own ROLLBACK, so an error return would +// not swap). Plain ctx cancellation never reaches this branch: the gate never +// binds a ctx to its connection's interrupt. +func TestWriteGate_SwapKeepsPreSwapConnectionOwned(t *testing.T) { + pool := NewPoolWithOptions(t.TempDir()+"/swap.sq3", PoolOptions{Size: 3, DisableAutoCheckpoint: true}) + armUngatedWriteCheck(t, pool) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + g, err := newWriteGate(ctx, pool) + require.NoError(t, err) + // No pool.Close here: the swapped-out connection is closed, never + // returned, and sqlitex.Pool.Close waits for it forever — the K-5 + // residual cleanOrReplace documents. + t.Cleanup(func() { require.NoError(t, g.Close()) }) + + c0 := g.conn + closed := make(chan struct{}) + close(closed) + err = g.run(ctx, priorityHigh, "swap", "test", func(_ context.Context, conn *sqlite.Conn) error { + require.Same(t, c0, conn) + require.NoError(t, sqlitex.ExecuteTransient(conn, `INSERT INTO metadata (kind,namespace,name,metadata) VALUES ('k','n','swap','{}')`, nil)) + conn.SetInterrupt(closed) + panic("forced failure after the recorded INSERT") + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "panic") + assert.Equal(t, 1, g.replaced, "the dirty connection must have been replaced") + assert.NotSame(t, c0, g.conn) + now := writeStmtSeq.Load() + assert.True(t, g.owns(c0, now), "the pre-swap connection stays owned: its recorded INSERT was a gated write") + // Nothing has been recorded on the replacement yet: the next write is the + // gate's (owns is bounded below by the take, WF-2). + assert.True(t, g.owns(g.conn, now+1), "the replacement is owned from its take on") + assert.False(t, g.owns(g.conn, now), "a write predating the replacement's take is not the gate's") + assert.False(t, g.held()) + + // The gate keeps working on the replacement, and the panicked INSERT + // never committed. + var n int64 + require.NoError(t, g.run(ctx, priorityHigh, "swap", "test", func(_ context.Context, conn *sqlite.Conn) error { + require.Same(t, g.conn, conn) + return sqlitex.ExecuteTransient(conn, `SELECT count(*) FROM metadata WHERE name='swap'`, &sqlitex.ExecOptions{ + ResultFunc: func(stmt *sqlite.Stmt) error { n = stmt.ColumnInt64(0); return nil }, + }) + })) + assert.Equal(t, int64(0), n) + // The armed AC-G1 check at cleanup is the CR-1 assertion: the INSERT on + // c0 must be judged owned, not flagged as ungated. +} diff --git a/pkg/registry/file/sqliteobject_gnp_test.go b/pkg/registry/file/sqliteobject_gnp_test.go new file mode 100644 index 000000000..46b8d65ed --- /dev/null +++ b/pkg/registry/file/sqliteobject_gnp_test.go @@ -0,0 +1,72 @@ +package file + +// §5.6 row 9 of .omc/plans/full-acid-storage-architecture.md: +// GeneratedNetworkPolicyStorage's full-spec ContainerProfile list must read +// through the ContainerProfile storage.Interface (the ObjectStore under the +// flag), not the default StorageImpl — whose get() would delete a row the +// ObjectStore owns when the file it expects is absent (and, guarded, refuses +// the key outright). Its knownservers read stays on the default instance. + +import ( + "context" + "testing" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/storage" +) + +// querierSpy counts the calls GeneratedNetworkPolicyStorage makes on the +// default StorageQuerier. +type querierSpy struct { + StorageQuerier + getList int + getByCluster int +} + +func (s *querierSpy) GetList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error { + s.getList++ + return s.StorageQuerier.GetList(ctx, key, opts, listObj) +} + +func (s *querierSpy) GetByCluster(ctx context.Context, apiVersion, kind string, listObj runtime.Object) error { + s.getByCluster++ + return s.StorageQuerier.GetByCluster(ctx, apiVersion, kind, listObj) +} + +const testGNPPrefix = "/spdx.softwarecomposition.kubescape.io/generatednetworkpolicies/" + +func TestGNP_ContainerProfileReadsGoThroughTheCPStore(t *testing.T) { + e := newObjectStoreEnv(t) + // The template carries workload-kind Deployment / workload-name coredns + // and status ready, so it is available and maps to "deployment-coredns". + e.create(e.plain("deployment-coredns-coredns-1111-2222")) + + spy := &querierSpy{StorageQuerier: e.legacy} + gnp := NewGeneratedNetworkPolicyStorage(spy, e.store) + + out := &softwarecomposition.GeneratedNetworkPolicy{} + require.NoError(t, gnp.Get(e.ctx, testGNPPrefix+e.ns+"/deployment-coredns", storage.GetOptions{}, out)) + require.Equal(t, "deployment-coredns", out.Name) + + list := &softwarecomposition.GeneratedNetworkPolicyList{} + require.NoError(t, gnp.GetList(e.ctx, testGNPPrefix+e.ns, storage.ListOptions{Predicate: storage.Everything}, list)) + require.Len(t, list.Items, 1) + require.Equal(t, "deployment-coredns", list.Items[0].Name) + + // Non-CP reads are unchanged: knownservers come from the default + // instance (once per Get/GetList); no CP list ever reaches it. + require.Equal(t, 2, spy.getByCluster, "knownservers must still be read through the default StorageQuerier") + require.Equal(t, 0, spy.getList, "no ContainerProfile list may reach the default StorageQuerier") + + // The guarded default instance still refuses the CP key: the old wiring + // (CP reads through the default instance) is not silently working. + cpList := &softwarecomposition.ContainerProfileList{} + err := e.legacy.GetList(e.ctx, testCPPrefix+e.ns, storage.ListOptions{ + ResourceVersion: softwarecomposition.ResourceVersionFullSpec, + Predicate: storage.SelectionPredicate{Limit: containerProfileListLimit}, + }, cpList) + require.Error(t, err) + require.Contains(t, err.Error(), "owned by the ContainerProfile SQLite backend") +} diff --git a/pkg/registry/file/sqliteobject_guard_test.go b/pkg/registry/file/sqliteobject_guard_test.go new file mode 100644 index 000000000..80f2d125b --- /dev/null +++ b/pkg/registry/file/sqliteobject_guard_test.go @@ -0,0 +1,222 @@ +package file + +import ( + "context" + "os" + "reflect" + "sync" + "sync/atomic" + "testing" + + helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/storage" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitex" +) + +// guardFs counts every payload-file operation the legacy store issues. +type guardFs struct { + afero.Fs + ops atomic.Int64 +} + +func (g *guardFs) Open(name string) (afero.File, error) { g.ops.Add(1); return g.Fs.Open(name) } +func (g *guardFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + g.ops.Add(1) + return g.Fs.OpenFile(name, flag, perm) +} +func (g *guardFs) Rename(o, n string) error { g.ops.Add(1); return g.Fs.Rename(o, n) } +func (g *guardFs) Remove(name string) error { g.ops.Add(1); return g.Fs.Remove(name) } +func (g *guardFs) Stat(name string) (os.FileInfo, error) { + g.ops.Add(1) + return g.Fs.Stat(name) +} + +// TestINV4_LegacyStoreRefusesContainerProfileKeys runs every full-object +// operation of the legacy StorageImpl (the guarded paths of design §5.6) on a +// containerprofile key while the ObjectStore owns the kind, and asserts: the +// ownership refusal (InternalError), ZERO statements on metadata / payloads / +// time_series, ZERO payload-file operations, the rows untouched — and that the +// metadata-only reads, which both backends agree on, are NOT refused. +func TestINV4_LegacyStoreRefusesContainerProfileKeys(t *testing.T) { + e := newObjectStoreEnv(t) + fs := &guardFs{Fs: e.legacyFs} + e.legacy.appFs = fs + + // Seed through the owner. + e.create(e.ts("r1", 1, helpersv1.Learning, helpersv1.Partial)) + e.tick() + base := e.mustGet(e.baseKey) + before := e.inspect(e.baseKey) + listKey := testCPPrefix + e.ns + cp := func() *softwarecomposition.ContainerProfile { return &softwarecomposition.ContainerProfile{} } + identity := identityTryUpdate + + type op struct { + name string + run func() error + } + ops := []op{ + {"Get(full)", func() error { return e.legacy.Get(e.ctx, e.baseKey, storage.GetOptions{}, cp()) }}, + {"GetList(fullSpec)", func() error { + return e.legacy.GetList(e.ctx, listKey, storage.ListOptions{ResourceVersion: softwarecomposition.ResourceVersionFullSpec, Recursive: true}, &softwarecomposition.ContainerProfileList{}) + }}, + {"Create", func() error { return e.legacy.Create(e.ctx, e.key("guard-new"), e.plain("guard-new"), nil, 0) }}, + {"GuaranteedUpdate", func() error { return e.legacy.GuaranteedUpdate(e.ctx, e.baseKey, cp(), false, nil, identity, nil) }}, + {"Delete", func() error { return e.legacy.Delete(e.ctx, e.baseKey, cp(), nil, nil, nil, storage.DeleteOptions{}) }}, + {"GetByNamespace", func() error { + return e.legacy.GetByNamespace(e.ctx, "spdx.softwarecomposition.kubescape.io", ContainerProfileKind, e.ns, &softwarecomposition.ContainerProfileList{}) + }}, + {"GetByCluster", func() error { + return e.legacy.GetByCluster(e.ctx, "spdx.softwarecomposition.kubescape.io", ContainerProfileKind, &softwarecomposition.ContainerProfileList{}) + }}, + {"appendGobObjectFromFile", func() error { + list := &softwarecomposition.ContainerProfileList{} + v := reflect.ValueOf(&list.Items).Elem() + return e.legacy.appendGobObjectFromFile(e.ctx, DefaultStorageRoot+e.baseKey+GobExt, v) + }}, + {"CreateWithConn", func() error { + return e.withLegacyConn(func(conn *sqlite.Conn) error { + return e.legacy.CreateWithConn(e.ctx, conn, e.key("guard-new2"), e.plain("guard-new2"), nil, 0) + }) + }}, + {"GuaranteedUpdateWithConn", func() error { + return e.withLegacyConn(func(conn *sqlite.Conn) error { + return e.legacy.GuaranteedUpdateWithConn(e.ctx, conn, e.baseKey, cp(), false, nil, identity, nil, "") + }) + }}, + } + // The flag-off write path (singleWriterEnabled=false) reaches + // CreateWithConn/GuaranteedUpdateWithConn through Create/GuaranteedUpdate. + ops = append(ops, + op{"Create(singleWriter=false)", func() error { + old := singleWriterEnabled + singleWriterEnabled = false + defer func() { singleWriterEnabled = old }() + return e.legacy.Create(e.ctx, e.key("guard-new3"), e.plain("guard-new3"), nil, 0) + }}, + op{"GuaranteedUpdate(singleWriter=false)", func() error { + old := singleWriterEnabled + singleWriterEnabled = false + defer func() { singleWriterEnabled = old }() + return e.legacy.GuaranteedUpdate(e.ctx, e.baseKey, cp(), false, nil, identity, nil) + }}, + ) + + // An observer connection: PRAGMA data_version changes whenever ANY other + // connection commits a change to the database — a write detector that does + // not depend on statement caching (the authorizer fires at prepare time). + observer, err := e.pool.Take(e.ctx) + require.NoError(t, err) + defer e.pool.Put(observer) + dataVersion := func() int64 { + var v int64 + require.NoError(t, sqlitex.ExecuteTransient(observer, `PRAGMA data_version`, &sqlitex.ExecOptions{ResultFunc: func(stmt *sqlite.Stmt) error { v = stmt.ColumnInt64(0); return nil }})) + return v + } + + // Step 1-L's statement observer: every sqlitex.Execute call site in this + // package reports its name at execution time (the authorizer only sees + // prepares). The design's INV-4 instrument. + var stmts []string + var stmtsMu sync.Mutex + setStmtObserver(func(site string) { stmtsMu.Lock(); stmts = append(stmts, site); stmtsMu.Unlock() }) + defer setStmtObserver(nil) + + for _, o := range ops { + t.Run(o.name, func(t *testing.T) { + fs.ops.Store(0) + dv := dataVersion() + stmtsMu.Lock() + stmts = nil + stmtsMu.Unlock() + mark := e.rec.mark() + err := o.run() + actions := e.rec.since(mark) + stmtsMu.Lock() + executed := append([]string(nil), stmts...) + stmtsMu.Unlock() + assert.Empty(t, executed, "%s: legacy call sites executed statements after refusal", o.name) + require.Error(t, err, "must refuse") + assert.True(t, apierrors.IsInternalError(err), "refusal must be an InternalError, got %T: %v", err, err) + assert.Contains(t, err.Error(), "owned by the ContainerProfile SQLite backend") + for _, a := range actions { + assert.Empty(t, a.table, "%s: statement on %s (%v) after refusal", o.name, a.table, a.op) + } + assert.Equal(t, dv, dataVersion(), "%s: the database was written after refusal", o.name) + assert.Equal(t, int64(0), fs.ops.Load(), "%s: payload-file operation after refusal", o.name) + assert.Equal(t, before, e.inspect(e.baseKey), "%s: rows changed", o.name) + }) + } + + t.Run("metadata-only reads are not refused", func(t *testing.T) { + out := cp() + require.NoError(t, e.legacy.Get(e.ctx, e.baseKey, storage.GetOptions{ResourceVersion: softwarecomposition.ResourceVersionMetadata}, out)) + assert.Equal(t, base.ResourceVersion, out.ResourceVersion) + assert.Equal(t, base.UID, out.UID) + list := &softwarecomposition.ContainerProfileList{} + require.NoError(t, e.legacy.GetList(e.ctx, listKey, storage.ListOptions{ResourceVersion: softwarecomposition.ResourceVersionMetadata, Recursive: true}, list)) + require.Len(t, list.Items, 1) + assert.Equal(t, base.Name, list.Items[0].Name) + n, err := e.legacy.Count(listKey) + require.NoError(t, err) + assert.Equal(t, int64(1), n) + }) + + t.Run("other kinds are served", func(t *testing.T) { + key := "/spdx.softwarecomposition.kubescape.io/sbomsyft/" + e.ns + "/img" + require.NoError(t, e.legacy.Create(e.ctx, key, &softwarecomposition.SBOMSyft{ObjectMeta: metav1.ObjectMeta{Name: "img", Namespace: e.ns}}, nil, 0)) + require.NoError(t, e.legacy.Get(e.ctx, key, storage.GetOptions{}, &softwarecomposition.SBOMSyft{})) + require.NoError(t, e.legacy.Delete(e.ctx, key, &softwarecomposition.SBOMSyft{}, nil, nil, nil, storage.DeleteOptions{})) + }) + + t.Run("guard removed serves CP again", func(t *testing.T) { + e.legacy.SetForeignKinds(nil) + defer e.legacy.SetForeignKinds(IsContainerProfileKind) + // The legacy full read of an ObjectStore row finds no gob file. It + // used to delete the shared row (PM-4) — which is why the guard + // exists; the rollback-safety guard's read fallback now serves it + // from payloads instead (see + // TestINV4_UnguardedLegacyFullReadNoLongerDeletesOwnedRow). The + // metadata read asserted here is unaffected either way. + require.NoError(t, e.legacy.Get(e.ctx, e.baseKey, storage.GetOptions{ResourceVersion: softwarecomposition.ResourceVersionMetadata}, cp())) + }) +} + +func (e *objectStoreEnv) withLegacyConn(fn func(conn *sqlite.Conn) error) error { + conn, err := e.pool.Take(context.Background()) + if err != nil { + return err + } + defer e.pool.Put(conn) + return fn(conn) +} + +// TestINV4_UnguardedLegacyFullReadNoLongerDeletesOwnedRow pins what the +// PM-4 hazard became: without the guard, a legacy full GET on an ObjectStore +// key used to delete the shared metadata row because the gob file is +// missing. Rollback-safety guard Part 2 closes that at this site by +// construction — such a key satisfies all four fallback conditions (a +// metadata row, rv non-NULL, not a time-series row, a payloads row), so the +// read is served from the payloads body and the row is left untouched. The +// guard still exists for every write-side operation. +func TestINV4_UnguardedLegacyFullReadNoLongerDeletesOwnedRow(t *testing.T) { + e := newObjectStoreEnv(t) + created := e.create(e.plain("pm4")) + e.legacy.SetForeignKinds(nil) + out := &softwarecomposition.ContainerProfile{} + err := e.legacy.Get(e.ctx, e.key("pm4"), storage.GetOptions{}, out) + assert.NoError(t, err, "Part 2: the fallback serves this key instead of self-repairing it") + assert.Equal(t, created.ResourceVersion, out.ResourceVersion, "stamped from the row's rv column") + assert.Equal(t, created.UID, out.UID, "stamped from the row's uid column") + row := e.inspect(e.key("pm4")) + assert.True(t, row.metaExists, "the unguarded legacy read no longer deletes the row (the PM-4 hazard)") + assert.True(t, row.payloadExists, "and the payloads row it was served from survives") + e.legacy.SetForeignKinds(IsContainerProfileKind) +} diff --git a/pkg/registry/file/sqliteobject_inv1_test.go b/pkg/registry/file/sqliteobject_inv1_test.go new file mode 100644 index 000000000..641fe74bb --- /dev/null +++ b/pkg/registry/file/sqliteobject_inv1_test.go @@ -0,0 +1,137 @@ +package file + +import ( + "context" + "reflect" + "sync" + "sync/atomic" + "testing" + + helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/kubescape/storage/pkg/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/storage" +) + +// recordingProcessor notes whether the gate was held when a callback ran. +type recordingProcessor struct { + inner Processor + gate *writeGate + mu sync.Mutex + calls []string +} + +func (r *recordingProcessor) note(what string) { + r.mu.Lock() + defer r.mu.Unlock() + if r.gate.held() { + r.calls = append(r.calls, what+" (GATE HELD)") + } else { + r.calls = append(r.calls, what) + } +} +func (r *recordingProcessor) AfterCreate(ctx context.Context, o runtime.Object) error { + r.note("AfterCreate") + return r.inner.AfterCreate(ctx, o) +} +func (r *recordingProcessor) PreSave(ctx context.Context, o runtime.Object) error { + r.note("PreSave") + return r.inner.PreSave(ctx, o) +} +func (r *recordingProcessor) SetStorage(s ContainerProfileStorage) { r.inner.SetStorage(s) } +func (r *recordingProcessor) TimeSeriesRowFor(o runtime.Object) (TimeSeriesRow, string, bool) { + r.note("TimeSeriesRowFor") + return r.inner.(TimeSeriesRowProvider).TimeSeriesRowFor(o) +} + +type recordingDispatcher struct { + inner eventDispatcher + gate *writeGate + mu sync.Mutex + calls []string +} + +func (r *recordingDispatcher) note(what string) { + r.mu.Lock() + defer r.mu.Unlock() + if r.gate.held() { + r.calls = append(r.calls, what+" (GATE HELD)") + } else { + r.calls = append(r.calls, what) + } +} +func (r *recordingDispatcher) Added(k string, m, o runtime.Object) { + r.note("Added") + r.inner.Added(k, m, o) +} +func (r *recordingDispatcher) Modified(k string, m, o runtime.Object) { + r.note("Modified") + r.inner.Modified(k, m, o) +} +func (r *recordingDispatcher) Deleted(k string, m runtime.Object) { + r.note("Deleted") + r.inner.Deleted(k, m) +} + +// TestINV1_GateHolderExecutesOnlySQL: between BEGIN IMMEDIATE and COMMIT on +// the gate's connection, the only activity in the process is SQL on that +// connection against metadata/payloads/time_series — no PreSave, no +// AfterCreate, no watch dispatch, no statement on any other connection (a pool +// take), and the store owns no per-key MapMutex at all. +func TestINV1_GateHolderExecutesOnlySQL(t *testing.T) { + e := newObjectStoreEnv(t) + rp := &recordingProcessor{inner: e.processor, gate: e.store.gate} + rd := &recordingDispatcher{inner: e.wd, gate: e.store.gate} + e.store.processor = rp + e.store.watchDispatcher = rd + + // MapMutex: assert absent by type. + st := reflect.TypeOf(ObjectStore{}) + for i := 0; i < st.NumField(); i++ { + assert.NotEqual(t, reflect.TypeOf(utils.MapMutex[string]{}), st.Field(i).Type, "ObjectStore must not carry a per-key MapMutex") + } + + var poolTakesUnderGate atomic.Int64 + var poolTakes atomic.Int64 + e.store.hooks.onPoolTake = func() { + poolTakes.Add(1) + if e.store.gate.held() { + poolTakesUnderGate.Add(1) + } + } + + mark := e.rec.mark() + // Every write path: TS create (with AfterCreate's row), REST update, a + // consolidation tick (write set), and a delete. + e.create(e.ts("r1", 1, helpersv1.Learning, helpersv1.Partial)) + require.NoError(t, e.store.GuaranteedUpdate(e.ctx, e.tsKey("r1"), &softwarecomposition.ContainerProfile{}, false, nil, setLabel("inv", "1"), nil)) + e.tick() + require.NoError(t, e.store.Delete(e.ctx, e.baseKey, nil, nil, nil, nil, storage.DeleteOptions{})) + actions := e.rec.since(mark) + + for _, c := range append(rp.calls, rd.calls...) { + assert.NotContains(t, c, "GATE HELD", "callback ran while the gate was held: %s", c) + } + assert.Contains(t, rp.calls, "PreSave") + assert.Contains(t, rd.calls, "Added") + assert.Contains(t, rd.calls, "Modified") + assert.Contains(t, rd.calls, "Deleted") + assert.NotContains(t, rp.calls, "AfterCreate", "the row provider path never calls AfterCreate") + + assert.Greater(t, poolTakes.Load(), int64(0)) + assert.Equal(t, int64(0), poolTakesUnderGate.Load(), "a pool connection was taken while the gate was held (acquisition order violated)") + + // The gate connection is used for nothing but the three tables. (The + // authorizer fires at prepare time; every statement the gate connection + // ever runs is prepared on it first, so the set of tables is exact.) + now := writeStmtSeq.Load() + for _, a := range actions { + if !e.store.gate.owns(a.conn, now) || a.table == "" { + continue + } + assert.Contains(t, []string{"metadata", "payloads", "time_series", "json_each"}, a.table, "unexpected table on the gate connection") + } +} diff --git a/pkg/registry/file/sqliteobject_inv2_rapid_test.go b/pkg/registry/file/sqliteobject_inv2_rapid_test.go new file mode 100644 index 000000000..449a51908 --- /dev/null +++ b/pkg/registry/file/sqliteobject_inv2_rapid_test.go @@ -0,0 +1,393 @@ +package file + +// INV-2 as a property: for every ContainerProfile key, at every point observable +// from a second connection, `metadata row exists ⇔ payloads row exists`, +// `rv == json_extract(metadata,'$.resourceVersion')` and +// `uid == json_extract(metadata,'$.uid')`. A rapid state machine generates +// Create / Update / Delete / TS-create / consolidation-tick sequences against +// the ObjectStore with a model of the expected key set, and a crash injector +// interrupts the gated transaction at EVERY statement boundary in three ways: +// returning an error (ROLLBACK), panicking (containment + ROLLBACK), and +// rolling the transaction back underneath the holder (what a process crash +// looks like to SQLite: the uncommitted WAL frames are discarded). After every +// step INV-2 is asserted from a fresh connection, along with the model. + +import ( + "errors" + "fmt" + "testing" + + helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/stretchr/testify/require" + "k8s.io/apiserver/pkg/storage" + "pgregory.net/rapid" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitex" +) + +type crashKind int + +const ( + crashNone crashKind = iota + crashError + crashPanic + crashRollbackUnderneath +) + +var errInjectedCrash = errors.New("injected crash") + +// inv2Machine is the rapid state machine. +type inv2Machine struct { + t *rapid.T + env *objectStoreEnv + // live is the model: keys the store must hold (base and TS), with the + // last RV observed for each. + live map[string]string + // crash configures the next gated transaction's interruption. + crashAt int + crashKind crashKind + armed bool + // tsBase remembers which TS suffixes have been created for the base. + tsSuffixes []string + names []string +} + +func (m *inv2Machine) armCrash(idx int, kind crashKind) { + m.crashAt, m.crashKind, m.armed = idx, kind, true +} + +// hook is the store's beforeStatement seam: fires the armed crash once. +func (m *inv2Machine) hook(conn *sqlite.Conn, path string, idx int, name string) error { + if !m.armed || idx != m.crashAt { + return nil + } + m.armed = false + switch m.crashKind { + case crashError: + return errInjectedCrash + case crashPanic: + panic(errInjectedCrash) + case crashRollbackUnderneath: + // The process "dies": SQLite discards the uncommitted WAL frames and + // nothing after this point runs. Emulated by rolling back on the + // holder's own connection AND aborting the holder (a first version + // let the remaining statements run in autocommit — which is not a + // crash, and did produce an orphan payload row). + _ = sqlitex.ExecuteTransient(conn, "ROLLBACK", nil) + return errInjectedCrash + } + return nil +} + +// disarm clears a crash that did not fire (fewer statements than crashAt). +func (m *inv2Machine) disarm() { m.armed = false } + +func (m *inv2Machine) checkINV2() { + m.env.withConn(func(conn *sqlite.Conn) { + keys := allCPKeys(m.env.t, conn) + for _, k := range keys { + assertINV2(m.env.t, conn, k) + } + // model: every live key is present, nothing else is + present := map[string]bool{} + for _, k := range keys { + present[k] = true + } + for k := range m.live { + if !present[k] { + m.t.Fatalf("model says %s is live but the store has no row", k) + } + } + for _, k := range keys { + if _, ok := m.live[k]; !ok { + m.t.Fatalf("store has %s but the model says it was never created or was deleted", k) + } + } + // rv agrees with the last observed RV + for k, rv := range m.live { + row := inspectRow(m.env.t, conn, k) + if row.jsonRV != rv { + m.t.Fatalf("%s: model rv %s, store rv %s", k, rv, row.jsonRV) + } + } + }) +} + +func (m *inv2Machine) maybeCrash(maxIdx int) bool { + if rapid.Bool().Draw(m.t, "crash") { + idx := rapid.IntRange(1, maxIdx).Draw(m.t, "crashAt") + kind := crashKind(rapid.IntRange(int(crashError), int(crashRollbackUnderneath)).Draw(m.t, "crashKind")) + m.armCrash(idx, kind) + return true + } + return false +} + +func (m *inv2Machine) pickName() string { + return m.names[rapid.IntRange(0, len(m.names)-1).Draw(m.t, "name")] +} + +// ---- actions ---- + +func (m *inv2Machine) Create(t *rapid.T) { + name := m.pickName() + key := m.env.key(name) + crashed := m.maybeCrash(4) + p := m.env.plain(name) + out := &softwarecomposition.ContainerProfile{} + err := m.env.store.Create(m.env.ctx, key, p, out, 0) + m.disarm() + _, existed := m.live[key] + switch { + case crashed && errors.Is(err, errInjectedCrash): + // interrupted: nothing changed + case existed: + if !storage.IsExist(err) { + t.Fatalf("create of existing %s: want KeyExists, got %v", key, err) + } + default: + if err != nil { + t.Fatalf("create %s: %v", key, err) + } + m.live[key] = out.ResourceVersion + } + m.checkINV2() +} + +func (m *inv2Machine) Update(t *rapid.T) { + name := m.pickName() + key := m.env.key(name) + crashed := m.maybeCrash(3) + out := &softwarecomposition.ContainerProfile{} + label := fmt.Sprintf("v%d", rapid.IntRange(0, 1000).Draw(t, "label")) + err := m.env.store.GuaranteedUpdate(m.env.ctx, key, out, false, nil, setLabel("l", label), nil) + m.disarm() + _, existed := m.live[key] + switch { + case crashed && errors.Is(err, errInjectedCrash): + // interrupted: rv unchanged + case !existed: + if !storage.IsNotFound(err) { + t.Fatalf("update of absent %s: want NotFound, got %v", key, err) + } + default: + if err != nil { + t.Fatalf("update %s: %v", key, err) + } + m.live[key] = out.ResourceVersion + } + m.checkINV2() +} + +func (m *inv2Machine) Delete(t *rapid.T) { + name := m.pickName() + key := m.env.key(name) + crashed := m.maybeCrash(4) + err := m.env.store.Delete(m.env.ctx, key, &softwarecomposition.ContainerProfile{}, nil, nil, nil, storage.DeleteOptions{}) + m.disarm() + _, existed := m.live[key] + switch { + case crashed && errors.Is(err, errInjectedCrash): + case !existed: + if !storage.IsNotFound(err) { + t.Fatalf("delete of absent %s: want NotFound, got %v", key, err) + } + default: + if err != nil { + t.Fatalf("delete %s: %v", key, err) + } + delete(m.live, key) + } + m.checkINV2() +} + +// CreateTS creates a time-series profile of the base (a chained report), so +// consolidation has something to merge; its row joins the transaction. +func (m *inv2Machine) CreateTS(t *rapid.T) { + n := len(m.tsSuffixes) + 1 + suffix := fmt.Sprintf("r%d", n) + key := m.env.tsKey(suffix) + crashed := m.maybeCrash(5) + out := &softwarecomposition.ContainerProfile{} + err := m.env.store.Create(m.env.ctx, key, m.env.ts(suffix, n, helpersv1.Learning, helpersv1.Partial), out, 0) + m.disarm() + switch { + case errors.Is(err, ObjectCompletedError): + // the base is Completed/Full: refused, nothing written + case crashed && errors.Is(err, errInjectedCrash): + default: + if err != nil { + t.Fatalf("create ts %s: %v", key, err) + } + m.live[key] = out.ResourceVersion + m.tsSuffixes = append(m.tsSuffixes, suffix) + } + m.checkINV2() +} + +// Tick runs one consolidation pass: the write set (base save + time_series +// rewrite + processed TS deletes) commits atomically or not at all. +func (m *inv2Machine) Tick(t *rapid.T) { + crashed := m.maybeCrash(4) + err := m.env.processor.ConsolidateTimeSeries(m.env.ctx) + m.disarm() + if err != nil && (!crashed || !errors.Is(err, errInjectedCrash)) { + t.Fatalf("tick: %v", err) + } + // Re-derive the model from the store: a committed tick deleted the + // processed TS objects and created/updated the base; an interrupted one + // changed nothing. Which happened is decided by whether the TS keys are + // still there — but INV-2 and "pre-tick or post-tick, nothing between" are + // what we assert: either ALL processed TS keys are gone and the base + // exists, or NONE are gone. + m.env.withConn(func(conn *sqlite.Conn) { + present := map[string]bool{} + for _, k := range allCPKeys(m.env.t, conn) { + present[k] = true + } + gone, kept := 0, 0 + for _, sfx := range m.tsSuffixes { + k := m.env.tsKey(sfx) + if _, live := m.live[k]; !live { + continue + } + if present[k] { + kept++ + } else { + gone++ + } + } + if gone > 0 && kept > 0 { + t.Fatalf("tick left a partial state: %d TS objects deleted, %d kept (crashed=%v err=%v)", gone, kept, crashed, err) + } + if gone > 0 { + if !present[m.env.baseKey] { + t.Fatalf("tick deleted TS objects but the base is absent") + } + for _, sfx := range m.tsSuffixes { + delete(m.live, m.env.tsKey(sfx)) + } + m.tsSuffixes = nil + m.live[m.env.baseKey] = inspectRow(m.env.t, conn, m.env.baseKey).jsonRV + } else if present[m.env.baseKey] { + m.live[m.env.baseKey] = inspectRow(m.env.t, conn, m.env.baseKey).jsonRV + } + }) + m.checkINV2() +} + +func (m *inv2Machine) Check(t *rapid.T) { + m.checkINV2() +} + +func TestINV2_RapidStateMachineWithCrashInjection(t *testing.T) { + rapid.Check(t, func(rt *rapid.T) { + e := newObjectStoreEnv(t) + m := &inv2Machine{t: rt, env: e, live: map[string]string{}, names: []string{"a", "b", "c"}} + e.store.hooks.beforeStatement = m.hook + // the base key must be part of the model once consolidation creates it + defer func() { e.store.hooks.beforeStatement = nil }() + rt.Repeat(rapid.StateMachineActions(m)) + }) +} + +// TestINV2_CrashAtEveryBoundaryDeterministic walks every statement boundary +// of the three transactions with each crash kind, deterministically, so the +// property has an exhaustive floor under the randomised machine. +func TestINV2_CrashAtEveryBoundaryDeterministic(t *testing.T) { + kinds := []crashKind{crashError, crashPanic, crashRollbackUnderneath} + for _, kind := range kinds { + for idx := 1; idx <= 6; idx++ { + t.Run(fmt.Sprintf("kind=%d/idx=%d", kind, idx), func(t *testing.T) { + e := newObjectStoreEnv(t) + m := &inv2Machine{env: e, live: map[string]string{}} + e.store.hooks.beforeStatement = m.hook + seedTS := e.ts("r1", 1, helpersv1.Learning, helpersv1.Partial) + + // Create (TS admission, insert-metadata, insert-payload, + // insert-time-series, commit = 5 boundaries; idx 6 never fires) + m.armCrash(idx, kind) + err := e.store.Create(e.ctx, e.tsKey("r1"), seedTS, nil, 0) + fired := !m.armed + m.disarm() + e.withConn(func(c *sqlite.Conn) { + for _, k := range allCPKeys(t, c) { + assertINV2(t, c, k) + } + }) + row := e.inspect(e.tsKey("r1")) + base := e.inspect(e.baseKey) + if fired { + require.ErrorIs(t, err, errInjectedCrash) + require.False(t, row.metaExists || row.payloadExists, "interrupted create left rows") + require.Equal(t, 0, base.tsRows, "interrupted create left a time_series row") + // re-create cleanly for the next transactions + require.NoError(t, e.store.Create(e.ctx, e.tsKey("r1"), e.ts("r1", 1, helpersv1.Learning, helpersv1.Partial), nil, 0)) + } else { + require.NoError(t, err) + require.True(t, row.metaExists && row.payloadExists) + require.Equal(t, 1, base.tsRows) + } + + // Update (update-metadata, update-payload, commit = 3 boundaries) + m.armCrash(idx, kind) + err = e.store.GuaranteedUpdate(e.ctx, e.tsKey("r1"), &softwarecomposition.ContainerProfile{}, false, nil, setLabel("x", "y"), nil) + fired = !m.armed + m.disarm() + e.withConn(func(c *sqlite.Conn) { assertINV2(t, c, e.tsKey("r1")) }) + got := e.mustGet(e.tsKey("r1")) + if fired { + require.ErrorIs(t, err, errInjectedCrash) + require.Equal(t, "1", got.ResourceVersion, "interrupted update must leave rv 1") + require.Empty(t, got.Labels["x"]) + } else { + require.NoError(t, err) + require.Equal(t, "2", got.ResourceVersion) + } + + // Consolidation tick: replace-time-series, save-base, delete-ts, + // commit = 4 boundaries + m.armCrash(idx, kind) + err = e.processor.ConsolidateTimeSeries(e.ctx) + fired = !m.armed + m.disarm() + e.withConn(func(c *sqlite.Conn) { + for _, k := range allCPKeys(t, c) { + assertINV2(t, c, k) + } + }) + tsRow := e.inspect(e.tsKey("r1")) + base = e.inspect(e.baseKey) + if fired { + require.ErrorIs(t, err, errInjectedCrash, "interrupted tick must report the error") + require.True(t, tsRow.metaExists, "pre-tick state: TS object still there") + require.False(t, base.metaExists, "pre-tick state: no base") + require.Equal(t, 1, base.tsRows, "pre-tick state: time_series row still there") + } else { + require.NoError(t, err) + require.False(t, tsRow.metaExists, "post-tick state: TS object deleted") + require.True(t, base.metaExists && base.payloadExists, "post-tick state: base created") + } + + // Delete of the base (delete-metadata, delete-payload, delete-time-series, commit) + if base.metaExists { + m.armCrash(idx, kind) + err = e.store.Delete(e.ctx, e.baseKey, nil, nil, nil, nil, storage.DeleteOptions{}) + fired = !m.armed + m.disarm() + e.withConn(func(c *sqlite.Conn) { assertINV2(t, c, e.baseKey) }) + after := e.inspect(e.baseKey) + if fired { + require.ErrorIs(t, err, errInjectedCrash) + require.True(t, after.metaExists && after.payloadExists, "interrupted delete must leave both rows") + } else { + require.NoError(t, err) + require.False(t, after.metaExists || after.payloadExists) + } + } + require.True(t, e.store.gate.conn.AutocommitEnabled(), "gate connection left dirty") + }) + } + } +} diff --git a/pkg/registry/file/sqliteobject_keyreserve.go b/pkg/registry/file/sqliteobject_keyreserve.go new file mode 100644 index 000000000..48fa7218b --- /dev/null +++ b/pkg/registry/file/sqliteobject_keyreserve.go @@ -0,0 +1,235 @@ +package file + +// Consolidation's fairness escalation against a same-series writer (design +// §3.7 Phase 3: "conflict → one re-read and retry, N=2"). +// +// The pass's Phase 1 read → merge → encode → gate → commit window is longer +// than a REST writer's whole read → encode → commit cycle, so a writer +// committing base updates back-to-back on the same key beats the pass's CAS +// on every attempt: the once-retry re-runs the same window and loses again, +// and nothing escalates across ticks. The gate's lanes do not help — a single +// writer never has a second commit queued while its first holds the gate, so +// the pass loses in its prepare window, outside the gate, not in the queue. +// +// The escalation is a per-series reservation the pass takes for its retry +// only, after a real conflict. While it is held, an ObjectStore write to the +// series (the base key or one of its TS keys) that has not started yet waits +// for the retry to end (bounded by keyReserveWaitMax), and the retry itself +// starts only once every same-series write already in flight has finished +// (bounded by keyReserveDrainMax). Absent those two timeouts, no same-series +// write commits inside the retry's window, so its CAS cannot fail from one: +// the series consolidates within the tick, at attempt 2. The reservation is +// released as soon as the retry ends, whichever way, so consolidation never +// holds writers across ticks; a writer whose wait times out proceeds anyway. +// The gate's priority scheme is untouched: writers to any other series are +// not affected at all. + +import ( + "context" + "strings" + "sync" + "time" + + "github.com/kubescape/storage/pkg/metrics" +) + +// Package-level so tests can shrink them. +var ( + // keyReserveWaitMax bounds an ordinary writer's wait on a series + // reservation in enter() (retry-vs-write: common, on every write to a + // contended series). Tier B (droplet run, b8e42f76) showed a 1s cap + // here shifting +47% onto update-p95-ms; a clean dedicated-CPU rerun + // at 200ms (commit d4e8090d) still showed +70.6%. 50ms fixed it + // (-9% to -10.5% across two runs) with no correctness cost: the + // starvation-fix regression tests still show consolidation reserving + // and committing every round via a normal release, never a timeout. + keyReserveWaitMax = 50 * time.Millisecond + // keyReserveQueueMax bounds one consolidation retry's wait to reserve + // a series another retry already holds, in reserve() (retry-vs-retry: + // overlapping ticks on the same series, rarer than every write). This + // used to share keyReserveWaitMax with the writer-wait above; shrinking + // that to 50ms for update-p95-ms made overlapping retries give up and + // run unreserved far more often, producing extra unconsolidated/ + // duplicate TS rows that List's scan/merge then paid for — list-p95-ms + // regressed +93.7% to +121.1% even though bumping the SQLite pool size + // (LOAD_POOL) made it worse, not better, ruling out pool contention as + // the cause. Split into its own constant, left generous like + // keyReserveDrainMax: retry-vs-retry collisions are uncommon enough + // that patiently waiting here costs little, and correctness (never + // racing two retries unreserved against each other) matters more than + // shaving this specific wait. + keyReserveQueueMax = time.Second + // keyReserveDrainMax bounds the reserving pass's wait for in-flight + // same-series writes to finish. Left at 1s: this is the tail-latency + // bound that fixed consolidation starvation and must stay generous. + keyReserveDrainMax = time.Second +) + +type keyReservationCtxKey struct{} + +// keyReservation is one series reserved by a consolidation retry. +type keyReservation struct { + key string + released chan struct{} + // drained is signalled (non-blocking) by every leave of a same-series + // write; the reserver re-counts on each signal. + drained chan struct{} + once sync.Once +} + +// keyReservations tracks reserved series and in-flight writes per key. +type keyReservations struct { + mu sync.Mutex + reserved map[string]*keyReservation + inflight map[string]int +} + +func newKeyReservations() *keyReservations { + return &keyReservations{reserved: map[string]*keyReservation{}, inflight: map[string]int{}} +} + +// sameSeries reports whether key is base itself or one of its TS keys +// (base + "-" + suffix, SplitProfileName's shape). A base named with a +// trailing "-x" segment matches another base's series by prefix; the cost is +// one bounded wait during that series' reserved retry, never a wrong write. +func sameSeries(key, base string) bool { + return key == base || (len(key) > len(base) && key[len(base)] == '-' && strings.HasPrefix(key, base)) +} + +// enter registers a write to key as in flight and returns its leave. When a +// reservation covers key's series, the write first waits for it to be +// released (bounded by keyReserveWaitMax and ctx); the reserving pass's own +// writes (ctx carries the reservation) never wait on it. +func (r *keyReservations) enter(ctx context.Context, key string) (leave func()) { + own, _ := ctx.Value(keyReservationCtxKey{}).(*keyReservation) + waited := false + for { + r.mu.Lock() + var blocking *keyReservation + for _, res := range r.reserved { + if res != own && sameSeries(key, res.key) { + blocking = res + break + } + } + if blocking == nil || waited { + r.inflight[key]++ + r.mu.Unlock() + return func() { r.leave(key) } + } + r.mu.Unlock() + + timer := time.NewTimer(keyReserveWaitMax) + select { + case <-blocking.released: + timer.Stop() + metrics.IncCPKeyYield(metrics.KeyYieldReleased) + case <-timer.C: + metrics.IncCPKeyYield(metrics.KeyYieldTimeout) + waited = true + case <-ctx.Done(): + timer.Stop() + // The write fails on its own ctx at the next step; register it so + // the leave stays uniform. + waited = true + } + } +} + +func (r *keyReservations) leave(key string) { + r.mu.Lock() + defer r.mu.Unlock() + if r.inflight[key] <= 1 { + delete(r.inflight, key) + } else { + r.inflight[key]-- + } + for _, res := range r.reserved { + if sameSeries(key, res.key) { + select { + case res.drained <- struct{}{}: + default: + } + } + } +} + +// reserve reserves key's series for the caller and waits for in-flight +// same-series writes to finish (bounded by keyReserveDrainMax and ctx). It +// returns the ctx the reserving pass must use for its own writes and the +// release, which is idempotent and must be called. drained is false when the +// drain timed out: writes were still in flight when the reservation began. +func (r *keyReservations) reserve(ctx context.Context, key string) (reservedCtx context.Context, release func(), drained bool) { + res := &keyReservation{key: key, released: make(chan struct{}), drained: make(chan struct{}, 1)} + for { + r.mu.Lock() + prev, taken := r.reserved[key] + if !taken { + r.reserved[key] = res + r.mu.Unlock() + break + } + r.mu.Unlock() + // Another pass reserved the same key (two overlapping ticks): queue + // behind it, bounded; past the bound the retry runs unreserved. + timer := time.NewTimer(keyReserveQueueMax) + select { + case <-prev.released: + timer.Stop() + case <-timer.C: + return ctx, func() {}, false + case <-ctx.Done(): + timer.Stop() + return ctx, func() {}, false + } + } + release = func() { r.mu.Lock(); res.close(r); r.mu.Unlock() } + + timer := time.NewTimer(keyReserveDrainMax) + defer timer.Stop() + drained = true + for r.inflightSameSeries(key) > 0 { + select { + case <-res.drained: + case <-timer.C: + return context.WithValue(ctx, keyReservationCtxKey{}, res), release, false + case <-ctx.Done(): + return context.WithValue(ctx, keyReservationCtxKey{}, res), release, false + } + } + return context.WithValue(ctx, keyReservationCtxKey{}, res), release, drained +} + +// close removes res from the map (if still registered) and releases its +// waiters. Caller holds r.mu. +func (res *keyReservation) close(r *keyReservations) { + res.once.Do(func() { + if r.reserved[res.key] == res { + delete(r.reserved, res.key) + } + close(res.released) + }) +} + +func (r *keyReservations) inflightSameSeries(base string) int { + r.mu.Lock() + defer r.mu.Unlock() + n := 0 + for key, c := range r.inflight { + if sameSeries(key, base) { + n += c + } + } + return n +} + +// reservedKeys reports the currently reserved series, for tests. +func (r *keyReservations) reservedKeys() []string { + r.mu.Lock() + defer r.mu.Unlock() + out := make([]string, 0, len(r.reserved)) + for k := range r.reserved { + out = append(out, k) + } + return out +} diff --git a/pkg/registry/file/sqliteobject_keyreserve_test.go b/pkg/registry/file/sqliteobject_keyreserve_test.go new file mode 100644 index 000000000..8de11e409 --- /dev/null +++ b/pkg/registry/file/sqliteobject_keyreserve_test.go @@ -0,0 +1,163 @@ +package file + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func shrinkKeyReserveBounds(t *testing.T, wait, drain time.Duration) { + t.Helper() + prevWait, prevDrain := keyReserveWaitMax, keyReserveDrainMax + keyReserveWaitMax, keyReserveDrainMax = wait, drain + t.Cleanup(func() { keyReserveWaitMax, keyReserveDrainMax = prevWait, prevDrain }) +} + +func TestSameSeries(t *testing.T) { + const base = "/prefix/ns/replicaset-nginx-6d4cf56db6-nginx" + require.True(t, sameSeries(base, base)) + require.True(t, sameSeries(base+"-3f9a", base)) + require.False(t, sameSeries(base+"3f9a", base)) + require.False(t, sameSeries("/prefix/ns/replicaset-nginx", base)) + require.False(t, sameSeries("/prefix/ns/other", base)) + require.False(t, sameSeries("/prefix/other/replicaset-nginx-6d4cf56db6-nginx", base)) +} + +// A write that starts while the series is reserved waits for the release; +// one that started before the reservation is waited for by the reserver. +func TestKeyReservations_WritersYieldAndReserverDrains(t *testing.T) { + shrinkKeyReserveBounds(t, 5*time.Second, 5*time.Second) + r := newKeyReservations() + const base = "/p/ns/base" + ctx := context.Background() + + // An in-flight writer on a TS key of the series. + leaveInflight := r.enter(ctx, base+"-r7") + + reserved := make(chan struct{}) + var reservedCtx context.Context + var release func() + var drained bool + go func() { + reservedCtx, release, drained = r.reserve(ctx, base) + close(reserved) + }() + select { + case <-reserved: + t.Fatal("reserve returned while a same-series write was in flight") + case <-time.After(50 * time.Millisecond): + } + leaveInflight() + select { + case <-reserved: + case <-time.After(2 * time.Second): + t.Fatal("reserve did not return after the in-flight write left") + } + require.True(t, drained) + require.Equal(t, []string{base}, r.reservedKeys()) + + // A new writer on the base key, and one on a TS key, block on the reservation. + entered := make(chan string, 2) + for _, k := range []string{base, base + "-r8"} { + go func(k string) { + leave := r.enter(ctx, k) + entered <- k + leave() + }(k) + } + select { + case k := <-entered: + t.Fatalf("writer on %s entered while the series was reserved", k) + case <-time.After(50 * time.Millisecond): + } + // A writer on another series is not affected. + leaveOther := r.enter(ctx, "/p/ns/other") + leaveOther() + // The reserver's own writes (ctx carries the reservation) do not wait. + leaveOwn := r.enter(reservedCtx, base) + leaveOwn() + + release() + for i := 0; i < 2; i++ { + select { + case <-entered: + case <-time.After(2 * time.Second): + t.Fatal("writer did not enter after the release") + } + } + require.Empty(t, r.reservedKeys()) + release() // idempotent +} + +// Both waits are bounded: a writer proceeds after keyReserveWaitMax even if +// the reservation is never released, and the reserver proceeds (drained=false) +// after keyReserveDrainMax even if an in-flight write never leaves. +func TestKeyReservations_BoundedWaits(t *testing.T) { + shrinkKeyReserveBounds(t, 30*time.Millisecond, 30*time.Millisecond) + r := newKeyReservations() + const base = "/p/ns/base" + ctx := context.Background() + + stuck := r.enter(ctx, base) + t0 := time.Now() + _, release, drained := r.reserve(ctx, base) + require.False(t, drained) + require.GreaterOrEqual(t, time.Since(t0), 30*time.Millisecond) + + t0 = time.Now() + leave := r.enter(ctx, base+"-r1") + require.GreaterOrEqual(t, time.Since(t0), 30*time.Millisecond) + leave() + release() + stuck() + require.Empty(t, r.reservedKeys()) +} + +// A cancelled writer ctx ends the wait at once. +func TestKeyReservations_WriterCtxCancel(t *testing.T) { + shrinkKeyReserveBounds(t, 5*time.Second, 5*time.Second) + r := newKeyReservations() + const base = "/p/ns/base" + _, release, _ := r.reserve(context.Background(), base) + defer release() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + done := make(chan struct{}) + go func() { r.enter(ctx, base)(); close(done) }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("a cancelled writer kept waiting on the reservation") + } +} + +// A second reserver of the same key queues behind the first. +func TestKeyReservations_SameKeyReserversQueue(t *testing.T) { + shrinkKeyReserveBounds(t, 5*time.Second, 5*time.Second) + r := newKeyReservations() + const base = "/p/ns/base" + ctx := context.Background() + _, release1, _ := r.reserve(ctx, base) + second := make(chan struct{}) + var release2 func() + go func() { + _, release2, _ = r.reserve(ctx, base) + close(second) + }() + select { + case <-second: + t.Fatal("second reserve returned while the first was held") + case <-time.After(50 * time.Millisecond): + } + release1() + select { + case <-second: + case <-time.After(2 * time.Second): + t.Fatal("second reserve did not return after the first released") + } + require.Equal(t, []string{base}, r.reservedKeys()) + release2() + require.Empty(t, r.reservedKeys()) +} diff --git a/pkg/registry/file/sqliteobject_migration.go b/pkg/registry/file/sqliteobject_migration.go new file mode 100644 index 000000000..37ebc5ece --- /dev/null +++ b/pkg/registry/file/sqliteobject_migration.go @@ -0,0 +1,843 @@ +package file + +// Startup data migration of ContainerProfile rows into the ObjectStore schema +// (design: .omc/plans/full-acid-storage-architecture.md §8.2, with K-1 and +// K-2's corrected reconcile rules). +// +// It runs synchronously in main.go, after the pool and the write gate exist +// and before the cleanup goroutine and the API server (R2: a concurrent +// cleanup tick could delete a row after its payload was copied). Every +// batch is one gated BEGIN IMMEDIATE … COMMIT containing only SQL on bytes +// prepared before the ticket (INV-1); the gob decode of a legacy file, the +// external migration tool and the JSON encode all run in the prepare phase +// on a pool connection that is released before the ticket is taken. +// +// The reconcile (steps 2 and 2b) runs on EVERY start: an older binary opens +// the migrated database without error and its INSERT OR REPLACE nulls rv/uid +// (R3), and its delete leaves the payloads row (K-2). The done-flag gates +// only the file sweeps (steps 3-4). Idempotent: a repaired row no longer +// matches the predicate. Resumable: a crash rolls back the batch in flight +// and the next start picks it up from the same predicate. + +import ( + "bytes" + "context" + "encoding/gob" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/kubescape/go-logger" + "github.com/kubescape/go-logger/helpers" + helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/kubescape/storage/pkg/metrics" + "github.com/spf13/afero" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/uuid" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitemigration" + "zombiezen.com/go/sqlite/sqlitex" +) + +// Reconcile shapes: the storage_cp_migration_total{shape} label values. +const ( + // MigrationShapeMigrated: a legacy row (rv NULL, no payloads row) whose + // file decoded and agrees with the row on resourceVersion. + MigrationShapeMigrated = "migrated" + // MigrationShapeDiverged: same, but the row and the file disagree on + // resourceVersion — one of the PC-DIV shapes, met once; the object gets + // max(rowRV, payloadRV)+1. + MigrationShapeDiverged = "diverged" + // MigrationShapeRowWithoutFile: a legacy row with no payload file — the + // row is deleted (today's get() self-repair, done once). + MigrationShapeRowWithoutFile = "row_without_file" + // MigrationShapeFileWithoutRow: a payload file with no metadata row (the + // file sweep) — imported at the file's resourceVersion and UID. + MigrationShapeFileWithoutRow = "file_without_row" + // MigrationShapeLegacyRewrite: rv NULL with a payloads row — a legacy + // writer replaced the row after a rollback (K-1). source=file when the + // legacy writer's .g file exists (the payloads body is the stale side); + // source=payloads for a row-only legacy write. + MigrationShapeLegacyRewrite = "legacy_rewrite" + // MigrationShapeOrphanPayload: a payloads row with no metadata row — a + // legacy delete after a rollback (K-2); deleted, or every Create of the + // key fails on the UNIQUE constraint forever. + MigrationShapeOrphanPayload = "orphan_payload" + // MigrationShapeUndecodable: a payload that neither gob nor the external + // tool could decode — skipped and counted, never deleted (PM-2). + MigrationShapeUndecodable = "undecodable" + // MigrationShapeTempFile: a *.g.t* staging file a legacy writer never + // committed — removed. + MigrationShapeTempFile = "temp_file" + + MigrationSourceFile = "file" + MigrationSourcePayloads = "payloads" + + migrationStateName = "containerprofile" + migrationStateDone = "done" + + // DefaultMigrationBatchSize is the number of objects per gated + // transaction (§8.2: B ≈ 50). + DefaultMigrationBatchSize = 50 +) + +// ContainerProfileMigrationOptions tunes MigrateContainerProfiles. +type ContainerProfileMigrationOptions struct { + // BatchSize is the number of objects per transaction; non-positive = + // DefaultMigrationBatchSize. + BatchSize int + // DryRun reconciles and counts without writing anything (§8.3): no + // gate is needed, no row, payload or file is touched, the done-flag is + // not set. + DryRun bool + + hooks migrationHooks +} + +// migrationHooks are test seams; nil in production. +type migrationHooks struct { + // beforeStatement runs inside the gated transaction of batch n before + // its statement idx (name labels it). Crash-injection tests return an + // error, panic, or exit the process from it. + beforeStatement func(batch, idx int, name string) error +} + +// ContainerProfileMigrationReport is what one run did. +type ContainerProfileMigrationReport struct { + // Counts is keyed by shape, or "shape/source" for legacy_rewrite. + Counts map[string]int + // Batches is the number of transactions committed (prepared, in dry-run). + Batches int + // SweepsRun reports whether the file sweeps (steps 3-4) ran this start. + SweepsRun bool + DryRun bool + Elapsed time.Duration +} + +func (r *ContainerProfileMigrationReport) add(shape, source string) { + key := shape + if source != "" { + key = shape + "/" + source + } + r.Counts[key]++ + if !r.DryRun { + metrics.IncCPMigration(shape, source) + } +} + +// Count returns the total for shape across sources. +func (r *ContainerProfileMigrationReport) Count(shape string) int { + n := r.Counts[shape] + for k, v := range r.Counts { + if strings.HasPrefix(k, shape+"/") { + n += v + } + } + return n +} + +// Work reports whether the run reconciled anything. +func (r *ContainerProfileMigrationReport) Work() int { + n := 0 + for _, v := range r.Counts { + n += v + } + return n +} + +func (r *ContainerProfileMigrationReport) logFields() []helpers.IDetails { + fields := []helpers.IDetails{ + helpers.Int("batches", r.Batches), + helpers.String("elapsed", r.Elapsed.String()), + helpers.Interface("sweepsRun", r.SweepsRun), + helpers.Interface("dryRun", r.DryRun), + } + for k, v := range r.Counts { + fields = append(fields, helpers.Int(k, v)) + } + return fields +} + +type containerProfileMigrator struct { + pool *sqlitemigration.Pool + gate *writeGate + fs afero.Fs + root string + scheme *runtime.Scheme + opts ContainerProfileMigrationOptions + report *ContainerProfileMigrationReport +} + +// migrationAction is one prepared write of a batch: SQL on bytes only. +type migrationAction struct { + name string + exec func(conn *sqlite.Conn) error +} + +// MigrateContainerProfiles reconciles every ContainerProfile row, payload +// and legacy file into the ObjectStore schema (see the file comment). gate +// may be nil only with opts.DryRun. fs/root are the legacy payload +// filesystem and its root (DefaultStorageRoot in production). +func MigrateContainerProfiles(ctx context.Context, pool *sqlitemigration.Pool, gate *WriteGate, fs afero.Fs, root string, scheme *runtime.Scheme, opts ContainerProfileMigrationOptions) (*ContainerProfileMigrationReport, error) { + if gate == nil && !opts.DryRun { + return nil, errors.New("containerprofile migration: a write gate is required (AC-G1: the migration's batches are gated writes)") + } + if opts.BatchSize <= 0 { + opts.BatchSize = DefaultMigrationBatchSize + } + start := time.Now() + // Cleaned once, as general hygiene. sweepFiles' key derivation no + // longer depends on root's exact textual form: see keyFromPayloadPath + // (same fix as ExportContainerProfiles, and for the same reason -- byte- + // length slicing at len(m.root) mis-derived the key for a trailing-slash + // root, and would have for "/" and "." too). + root = filepath.Clean(root) + m := &containerProfileMigrator{ + pool: pool, gate: gate, fs: fs, root: root, scheme: scheme, opts: opts, + report: &ContainerProfileMigrationReport{Counts: map[string]int{}, DryRun: opts.DryRun}, + } + if err := m.reconcileRows(ctx); err != nil { + return m.report, err + } + if err := m.reconcileOrphanPayloads(ctx); err != nil { + return m.report, err + } + done, err := m.readDone(ctx) + if err != nil { + return m.report, err + } + if !done { + m.report.SweepsRun = true + undecodable, err := m.sweepFiles(ctx) + if err != nil { + return m.report, err + } + // An undecodable file is left in place and counted; the sweep runs + // again next start so it stays visible until an operator acts. + if !opts.DryRun && undecodable == 0 { + if err := m.markDone(ctx); err != nil { + return m.report, err + } + } + } + if !opts.DryRun { + if err := m.checkpoint(ctx); err != nil { + return m.report, err + } + } + m.report.Elapsed = time.Since(start) + logger.L().Debug("containerprofile migration: done", m.report.logFields()...) + return m.report, nil +} + +// ---- step 2: rows with rv NULL or no payloads row ---- + +type migrationCandidate struct { + rowid int64 + namespace string + name string + metadataJSON []byte + rvNull bool + hasPayload bool + encoding string + body []byte +} + +func (m *containerProfileMigrator) readCandidates(ctx context.Context, cursor int64) ([]migrationCandidate, error) { + conn, err := m.pool.Take(ctx) + if err != nil { + return nil, fmt.Errorf("containerprofile migration: take connection: %w", err) + } + defer m.pool.Put(conn) + var out []migrationCandidate + err = sqlitex.Execute(conn, + `SELECT m.rowid, m.namespace, m.name, m.metadata, m.rv IS NULL, p.encoding IS NOT NULL, p.encoding, p.body + FROM metadata m LEFT JOIN payloads p USING (kind, namespace, name) + WHERE m.kind = :kind AND m.rowid > :cursor AND (m.rv IS NULL OR p.encoding IS NULL) + ORDER BY m.rowid LIMIT :limit`, + &sqlitex.ExecOptions{ + Named: map[string]any{":kind": ContainerProfileKind, ":cursor": cursor, ":limit": m.opts.BatchSize}, + ResultFunc: func(stmt *sqlite.Stmt) error { + c := migrationCandidate{ + rowid: stmt.ColumnInt64(0), + namespace: stmt.ColumnText(1), + name: stmt.ColumnText(2), + metadataJSON: []byte(stmt.ColumnText(3)), + rvNull: stmt.ColumnInt64(4) == 1, + hasPayload: stmt.ColumnInt64(5) == 1, + encoding: stmt.ColumnText(6), + } + if c.hasPayload { + c.body = make([]byte, stmt.ColumnLen(7)) + stmt.ColumnBytes(7, c.body) + } + out = append(out, c) + return nil + }, + }) + if err != nil { + return nil, fmt.Errorf("containerprofile migration: read candidates: %w", err) + } + return out, nil +} + +func (m *containerProfileMigrator) reconcileRows(ctx context.Context) error { + cursor := int64(0) + for { + if err := ctx.Err(); err != nil { + return err + } + cands, err := m.readCandidates(ctx, cursor) + if err != nil { + return err + } + if len(cands) == 0 { + return nil + } + cursor = cands[len(cands)-1].rowid + var actions []migrationAction + for i := range cands { + actions = append(actions, m.prepareCandidate(ctx, &cands[i])...) + } + if err := m.commit(ctx, actions); err != nil { + return err + } + } +} + +func (m *containerProfileMigrator) key(namespace, name string) string { + return K8sKeysToPath("", softwarecomposition.GroupName, ContainerProfileKind, "", namespace, name) +} + +func parseRV(s string) int64 { + rv, err := strconv.ParseInt(s, 10, 64) + if err != nil || rv < 0 { + return 0 + } + return rv +} + +// prepareCandidate decides and prepares one row's reconcile; it returns the +// SQL to run under the gate (none when the row is skipped). +func (m *containerProfileMigrator) prepareCandidate(ctx context.Context, c *migrationCandidate) []migrationAction { + key := m.key(c.namespace, c.name) + row := &PartialObjectMetadata{} + if err := json.Unmarshal(c.metadataJSON, row); err != nil { + m.report.add(MigrationShapeUndecodable, "row") + logger.L().Warning("containerprofile migration: metadata row is not JSON; skipped", helpers.Error(err), helpers.String("key", key)) + return nil + } + rowRV := parseRV(row.ResourceVersion) + obj, found, err := m.decodeLegacyFile(ctx, key) + if err != nil { + m.report.add(MigrationShapeUndecodable, MigrationSourceFile) + logger.L().Warning("containerprofile migration: payload file undecodable; row left as is, file left for the export tool", helpers.Error(err), helpers.String("key", key)) + return nil + } + + if !c.hasPayload { + if !found { + m.report.add(MigrationShapeRowWithoutFile, "") + logger.L().Warning("containerprofile migration: row without payload file; row deleted", helpers.String("key", key)) + return []migrationAction{m.deleteRowAction(key)} + } + payloadRV := parseRV(obj.ResourceVersion) + rv := rowRV + shape := MigrationShapeMigrated + switch { + case rowRV != payloadRV: + rv = max(rowRV, payloadRV) + 1 + shape = MigrationShapeDiverged + case rv == 0: + rv = 1 + } + // PreSave's non-TS revert (processor.go: a Completed base never + // regresses to Learning): the row's Completed beats the file's Learning. + if obj.Annotations[helpersv1.ReportSeriesIdMetadataKey] == "" && + row.Annotations[helpersv1.StatusMetadataKey] == helpersv1.Completed && + obj.Annotations[helpersv1.StatusMetadataKey] == helpersv1.Learning { + obj.Annotations[helpersv1.StatusMetadataKey] = helpersv1.Completed + } + uid := obj.UID + if uid == "" { + uid = row.UID + } + m.report.add(shape, "") + if shape != MigrationShapeMigrated { + logger.L().Warning("containerprofile migration: row and payload file disagree on resourceVersion; reconciled at max+1", + helpers.String("key", key), helpers.Interface("rowRV", rowRV), helpers.Interface("payloadRV", payloadRV), helpers.Interface("rv", rv)) + } + return m.rewriteActions(key, obj, rv, uid, true) + } + + // rv NULL with a payloads row: K-1's legacy_rewrite. + source := MigrationSourcePayloads + if found { + source = MigrationSourceFile + } else { + obj = &softwarecomposition.ContainerProfile{} + if err := decodePayloadBody(m.scheme, c.encoding, c.body, obj); err != nil { + m.report.add(MigrationShapeUndecodable, MigrationSourcePayloads) + logger.L().Warning("containerprofile migration: payloads body undecodable; row left as is", helpers.Error(err), helpers.String("key", key)) + return nil + } + } + // rv := the row JSON's resourceVersion, no +1 (the legacy writer already + // bumped it). When a crash left the legacy writer's file one version + // ahead of its row, or a stale pre-migration file sits behind a newer + // payloads body, the largest persisted version wins so no client sees a + // version regress; the row JSON is rewritten to match (INV-2). + rv := max(rowRV, parseRV(obj.ResourceVersion)) + if found { + bodyObj := &softwarecomposition.ContainerProfile{} + if err := decodePayloadBody(m.scheme, c.encoding, c.body, bodyObj); err == nil { + rv = max(rv, parseRV(bodyObj.ResourceVersion)) + } + } + if rv == 0 { + rv = 1 + } + uid := row.UID + if uid == "" { + uid = obj.UID + } + m.report.add(MigrationShapeLegacyRewrite, source) + logger.L().Warning("containerprofile migration: legacy writer replaced a migrated row (rv NULL with a payloads row); repaired", + helpers.String("key", key), helpers.String("source", source), helpers.Interface("rv", rv)) + return m.rewriteActions(key, obj, rv, uid, false) +} + +// rewriteActions stamps obj at (rv, uid), encodes the metadata JSON and the +// body, and returns the UPDATE of the metadata row plus the INSERT (fresh +// payload) or UPDATE (existing payload) of the payloads row. +func (m *containerProfileMigrator) rewriteActions(key string, obj *softwarecomposition.ContainerProfile, rv int64, uid types.UID, insertPayload bool) []migrationAction { + if uid == "" { + uid = uuid.NewUUID() + } + obj.ResourceVersion = strconv.FormatInt(rv, 10) + obj.UID = uid + metadataJSON, err := json.Marshal(extractFields(obj, []string{"ObjectMeta", "SchemaVersion"})) + if err != nil { + m.report.add(MigrationShapeUndecodable, "encode") + logger.L().Warning("containerprofile migration: marshal metadata failed; skipped", helpers.Error(err), helpers.String("key", key)) + return nil + } + body, err := encodePayloadBody(m.scheme, obj) + if err != nil { + m.report.add(MigrationShapeUndecodable, "encode") + logger.L().Warning("containerprofile migration: encode payload failed; skipped", helpers.Error(err), helpers.String("key", key)) + return nil + } + _, _, kind, _, namespace, name := K8sPathToKeys(key) + actions := []migrationAction{{ + name: "update-metadata", + exec: func(conn *sqlite.Conn) error { + if err := sqlitex.Execute(conn, + `UPDATE metadata SET metadata = ?, rv = ?, uid = ? WHERE kind = ? AND namespace = ? AND name = ?`, + &sqlitex.ExecOptions{Args: []any{string(metadataJSON), rv, string(uid), kind, namespace, name}}); err != nil { + return fmt.Errorf("update metadata %s: %w", key, err) + } + if n := conn.Changes(); n != 1 { + return fmt.Errorf("update metadata %s: %d rows", key, n) + } + return nil + }, + }} + if insertPayload { + actions = append(actions, migrationAction{ + name: "insert-payload", + exec: func(conn *sqlite.Conn) error { + return execInsertPayload(conn, key, kind, namespace, name, body) + }, + }) + } else { + actions = append(actions, migrationAction{ + name: "update-payload", + exec: func(conn *sqlite.Conn) error { + if err := sqlitex.Execute(conn, + `UPDATE payloads SET encoding = ?, body = ? WHERE kind = ? AND namespace = ? AND name = ?`, + &sqlitex.ExecOptions{Args: []any{PayloadEncodingJSONV1Beta1, body, kind, namespace, name}}); err != nil { + return fmt.Errorf("update payload %s: %w", key, err) + } + if n := conn.Changes(); n != 1 { + return fmt.Errorf("update payload %s: %d rows", key, n) + } + return nil + }, + }) + } + return actions +} + +func execInsertPayload(conn *sqlite.Conn, key, kind, namespace, name string, body []byte) error { + if err := sqlitex.Execute(conn, + `INSERT INTO payloads (kind, namespace, name, encoding, body) VALUES (?, ?, ?, ?, ?)`, + &sqlitex.ExecOptions{Args: []any{kind, namespace, name, PayloadEncodingJSONV1Beta1, body}}); err != nil { + return fmt.Errorf("insert payload %s: %w", key, err) + } + return nil +} + +func (m *containerProfileMigrator) deleteRowAction(key string) migrationAction { + _, _, kind, _, namespace, name := K8sPathToKeys(key) + return migrationAction{ + name: "delete-metadata", + exec: func(conn *sqlite.Conn) error { + if err := sqlitex.Execute(conn, + `DELETE FROM metadata WHERE kind = ? AND namespace = ? AND name = ?`, + &sqlitex.ExecOptions{Args: []any{kind, namespace, name}}); err != nil { + return fmt.Errorf("delete metadata %s: %w", key, err) + } + return nil + }, + } +} + +// decodeLegacyFile decodes the legacy gob payload of key. found is false +// when there is no file; err is set when the file exists but neither gob +// nor the external migration tool could decode it. +func (m *containerProfileMigrator) decodeLegacyFile(ctx context.Context, key string) (*softwarecomposition.ContainerProfile, bool, error) { + return decodeLegacyFileAt(ctx, m.fs, m.root, key) +} + +// errLegacyFileAccess wraps a decodeLegacyFileAt failure that is a real +// filesystem access problem (permission denied, I/O error) rather than the +// file's content being unreadable as a ContainerProfile. A caller that must +// tell the two apart -- content it can safely leave alone vs. an access +// failure that leaves it unable to say whether a file is safe to act on -- +// checks for this with errors.Is. +var errLegacyFileAccess = errors.New("legacy file access") + +// decodeLegacyFileAt is decodeLegacyFile without a migrator receiver, for +// callers (the export tool's stale-file reconciliation) that need the same +// "would an old binary read this as a valid object" decode, outside of a +// migration run. +func decodeLegacyFileAt(ctx context.Context, fs afero.Fs, root, key string) (*softwarecomposition.ContainerProfile, bool, error) { + p := filepath.Join(root, key) + f, err := fs.Open(makePayloadPath(p)) + if err != nil { + if errors.Is(err, os.ErrNotExist) || errors.Is(err, afero.ErrFileNotFound) { + return nil, false, nil + } + return nil, true, fmt.Errorf("open payload file: %w: %w", errLegacyFileAccess, err) + } + // Read fully before decoding, not decode straight off f: a Read error + // after a successful Open (a bad sector, a device/mount that opens fine + // but errors mid-read) would otherwise surface only inside gob's Decode + // as an opaque, untagged error -- indistinguishable from the file's + // content genuinely being malformed. Read errors are exactly as + // uncertain as Open errors (we cannot tell whether this object is safe + // to treat as absent/undecodable), so they get the same errLegacyFileAccess + // classification, not silently folded into "undecodable, safe to leave". + data, rerr := io.ReadAll(f) + _ = f.Close() + if rerr != nil { + return nil, true, fmt.Errorf("read payload file: %w: %w", errLegacyFileAccess, rerr) + } + obj := &softwarecomposition.ContainerProfile{} + err = gob.NewDecoder(bytes.NewReader(data)).Decode(obj) + if err == nil { + return obj, true, nil + } + if strings.Contains(err.Error(), "gob: wrong type") || strings.Contains(err.Error(), "extra fields") { + // The last time the external tool runs for this kind. + out, terr := execMigrationTool(ctx, p, "ContainerProfile") + if terr != nil { + // The tool's own exec/timeout/read failure is likewise not + // evidence the content is bad -- it can be a transient + // environment problem (missing binary, permission, timeout), + // so it gets the same errLegacyFileAccess classification as a + // direct file-access failure. + return nil, true, fmt.Errorf("gob decode: %v; migration tool: %w: %w", err, errLegacyFileAccess, terr) + } + obj = &softwarecomposition.ContainerProfile{} + if jerr := json.Unmarshal(out, obj); jerr != nil { + return nil, true, fmt.Errorf("gob decode: %v; migration tool output: %w", err, jerr) + } + return obj, true, nil + } + return nil, true, fmt.Errorf("gob decode: %w", err) +} + +// ---- step 2b: payloads rows with no metadata row (K-2) ---- + +func (m *containerProfileMigrator) reconcileOrphanPayloads(ctx context.Context) error { + cursor := int64(0) + for { + if err := ctx.Err(); err != nil { + return err + } + conn, err := m.pool.Take(ctx) + if err != nil { + return fmt.Errorf("containerprofile migration: take connection: %w", err) + } + var keys []string + err = sqlitex.Execute(conn, + `SELECT p.rowid, p.namespace, p.name FROM payloads p + WHERE p.kind = :kind AND p.rowid > :cursor AND NOT EXISTS ( + SELECT 1 FROM metadata m WHERE m.kind = p.kind AND m.namespace = p.namespace AND m.name = p.name) + ORDER BY p.rowid LIMIT :limit`, + &sqlitex.ExecOptions{ + Named: map[string]any{":kind": ContainerProfileKind, ":cursor": cursor, ":limit": m.opts.BatchSize}, + ResultFunc: func(stmt *sqlite.Stmt) error { + cursor = stmt.ColumnInt64(0) + keys = append(keys, m.key(stmt.ColumnText(1), stmt.ColumnText(2))) + return nil + }, + }) + m.pool.Put(conn) + if err != nil { + return fmt.Errorf("containerprofile migration: read orphan payloads: %w", err) + } + if len(keys) == 0 { + return nil + } + var actions []migrationAction + for _, key := range keys { + m.report.add(MigrationShapeOrphanPayload, "") + logger.L().Warning("containerprofile migration: payloads row without a metadata row (a legacy delete after a rollback); deleted", helpers.String("key", key)) + _, _, kind, _, namespace, name := K8sPathToKeys(key) + actions = append(actions, migrationAction{ + name: "delete-payload", + exec: func(conn *sqlite.Conn) error { + if err := sqlitex.Execute(conn, + `DELETE FROM payloads WHERE kind = ? AND namespace = ? AND name = ?`, + &sqlitex.ExecOptions{Args: []any{kind, namespace, name}}); err != nil { + return fmt.Errorf("delete payload %s: %w", key, err) + } + return nil + }, + }) + } + if err := m.commit(ctx, actions); err != nil { + return err + } + } +} + +// ---- steps 3-4: the file sweeps, gated by the done-flag ---- + +func (m *containerProfileMigrator) readDone(ctx context.Context) (bool, error) { + conn, err := m.pool.Take(ctx) + if err != nil { + return false, fmt.Errorf("containerprofile migration: take connection: %w", err) + } + defer m.pool.Put(conn) + var state string + err = sqlitex.Execute(conn, `SELECT state FROM migration_state WHERE name = ?`, &sqlitex.ExecOptions{ + Args: []any{migrationStateName}, + ResultFunc: func(stmt *sqlite.Stmt) error { state = stmt.ColumnText(0); return nil }, + }) + if err != nil { + return false, fmt.Errorf("containerprofile migration: read state: %w", err) + } + return state == migrationStateDone, nil +} + +func (m *containerProfileMigrator) markDone(ctx context.Context) error { + counts, _ := json.Marshal(m.report.Counts) + return m.commit(ctx, []migrationAction{{ + name: "mark-done", + exec: func(conn *sqlite.Conn) error { + return sqlitex.Execute(conn, + `INSERT OR REPLACE INTO migration_state (name, state, counts, updated_at) VALUES (?, ?, ?, ?)`, + &sqlitex.ExecOptions{Args: []any{migrationStateName, migrationStateDone, string(counts), time.Now().UTC().Format(time.RFC3339)}}) + }, + }}) +} + +// sweepFiles imports every payload file with no metadata row and removes +// every staging file. It returns the number of undecodable files met. +func (m *containerProfileMigrator) sweepFiles(ctx context.Context) (int, error) { + dir := filepath.Join(m.root, softwarecomposition.GroupName, ContainerProfileKind) + // A real filesystem error here (e.g. permission denied) must not read as + // "the directory doesn't exist": DirExists returns exists=false on ANY + // stat error, not just os.ErrNotExist, and the caller treats a nil error + // here as "sweep found nothing, mark done" -- an unreadable directory + // would then permanently persist the done marker over an incomplete + // sweep, with nothing left to trigger a retry on the next start. + exists, err := afero.DirExists(m.fs, dir) + if err != nil { + return 0, fmt.Errorf("containerprofile migration: stat %s: %w", dir, err) + } + if !exists { + return 0, nil + } + conn, err := m.pool.Take(ctx) + if err != nil { + return 0, fmt.Errorf("containerprofile migration: take connection: %w", err) + } + var orphans []string + walkErr := afero.Walk(m.fs, dir, func(path string, info os.FileInfo, err error) error { + // A per-entry error (e.g. an unreadable subtree) must abort the walk, + // not be treated as "nothing interesting here": returning nil let the + // sweep silently skip whatever orphan payload files sat under that + // entry and still report success, persisting the done marker over an + // incomplete sweep (see the DirExists check above for the same class + // of bug at the top-level directory). + if err != nil { + return fmt.Errorf("containerprofile migration: walk %s: %w", path, err) + } + if info.IsDir() { + return nil + } + if !IsPayloadFile(path) { + // The legacy staging names are .g.t (saveObject) and + // .g.t.. (the single writer); neither ends in + // .g, whereas a payload of a workload named e.g. "app.g.tail" does + // — the payload test above must come first (a bare ".g.t" + // substring match would delete that payload as a staging file). + if isLegacyStagingFile(info.Name()) { + m.report.add(MigrationShapeTempFile, "") + if !m.opts.DryRun { + if rerr := m.fs.Remove(path); rerr != nil { + logger.L().Warning("containerprofile migration: remove staging file failed", helpers.Error(rerr), helpers.String("path", path)) + } + } + } + return nil + } + key, kerr := keyFromPayloadPath(m.root, path) + if kerr != nil { + return fmt.Errorf("containerprofile migration: %w", kerr) + } + if _, rerr := ReadMetadata(conn, key); rerr == nil { + return nil + } else if !errors.Is(rerr, ErrMetadataNotFound) { + return fmt.Errorf("read metadata %s: %w", key, rerr) + } + orphans = append(orphans, key) + return nil + }) + m.pool.Put(conn) + if walkErr != nil { + return 0, fmt.Errorf("containerprofile migration: sweep %s: %w", dir, walkErr) + } + + undecodable := 0 + for start := 0; start < len(orphans); start += m.opts.BatchSize { + if err := ctx.Err(); err != nil { + return undecodable, err + } + end := min(start+m.opts.BatchSize, len(orphans)) + var actions []migrationAction + for _, key := range orphans[start:end] { + obj, found, err := m.decodeLegacyFile(ctx, key) + if err != nil || !found { + undecodable++ + m.report.add(MigrationShapeUndecodable, MigrationSourceFile) + logger.L().Warning("containerprofile migration: payload file without a row is undecodable; left for the export tool", helpers.Error(err), helpers.String("key", key)) + continue + } + m.report.add(MigrationShapeFileWithoutRow, "") + logger.L().Warning("containerprofile migration: payload file without a metadata row; imported", helpers.String("key", key)) + actions = append(actions, m.importActions(key, obj)...) + } + if err := m.commit(ctx, actions); err != nil { + return undecodable, err + } + } + return undecodable, nil +} + +// isLegacyStagingFile reports whether name is a legacy writer's staging +// file: .g.t (saveObject) or .g.t.. (the single +// writer). Callers exclude payload files (*.g) first. +func isLegacyStagingFile(name string) bool { + return strings.HasSuffix(name, GobExt+".t") || strings.Contains(name, GobExt+".t.") +} + +// importActions creates the metadata and payloads rows of a file with no row, +// at the file's resourceVersion and UID. +func (m *containerProfileMigrator) importActions(key string, obj *softwarecomposition.ContainerProfile) []migrationAction { + rv := max(parseRV(obj.ResourceVersion), 1) + uid := obj.UID + if uid == "" { + uid = uuid.NewUUID() + } + obj.ResourceVersion = strconv.FormatInt(rv, 10) + obj.UID = uid + metadataJSON, err := json.Marshal(extractFields(obj, []string{"ObjectMeta", "SchemaVersion"})) + if err != nil { + logger.L().Warning("containerprofile migration: marshal metadata failed; skipped", helpers.Error(err), helpers.String("key", key)) + return nil + } + body, err := encodePayloadBody(m.scheme, obj) + if err != nil { + logger.L().Warning("containerprofile migration: encode payload failed; skipped", helpers.Error(err), helpers.String("key", key)) + return nil + } + _, _, kind, _, namespace, name := K8sPathToKeys(key) + return []migrationAction{ + { + name: "insert-metadata", + exec: func(conn *sqlite.Conn) error { + if err := sqlitex.Execute(conn, + `INSERT INTO metadata (kind, namespace, name, metadata, rv, uid) VALUES (?, ?, ?, ?, ?, ?)`, + &sqlitex.ExecOptions{Args: []any{kind, namespace, name, string(metadataJSON), rv, string(uid)}}); err != nil { + return fmt.Errorf("insert metadata %s: %w", key, err) + } + return nil + }, + }, + { + name: "insert-payload", + exec: func(conn *sqlite.Conn) error { + return execInsertPayload(conn, key, kind, namespace, name, body) + }, + }, + } +} + +// ---- the gated batch ---- + +// commit runs actions in one gated transaction (nothing in dry-run). +func (m *containerProfileMigrator) commit(ctx context.Context, actions []migrationAction) error { + if len(actions) == 0 { + return nil + } + m.report.Batches++ + if m.opts.DryRun { + return nil + } + batch := m.report.Batches + err := m.gate.run(ctx, priorityLow, holdPathCPMigration, ContainerProfileKindPlural, func(_ context.Context, conn *sqlite.Conn) error { + for i, a := range actions { + if m.opts.hooks.beforeStatement != nil { + if err := m.opts.hooks.beforeStatement(batch, i, a.name); err != nil { + return err + } + } + if err := a.exec(conn); err != nil { + return err + } + } + return nil + }) + if err != nil { + return fmt.Errorf("containerprofile migration: batch %d rolled back: %w", batch, err) + } + return nil +} + +// checkpoint runs one PASSIVE checkpoint so the server does not start with +// the migration's whole WAL (wal_autocheckpoint is 0 on every connection). +func (m *containerProfileMigrator) checkpoint(ctx context.Context) error { + conn, err := m.pool.Take(ctx) + if err != nil { + return fmt.Errorf("containerprofile migration: take connection: %w", err) + } + defer m.pool.Put(conn) + if err := sqlitex.ExecuteTransient(conn, `PRAGMA wal_checkpoint(PASSIVE)`, nil); err != nil { + return fmt.Errorf("containerprofile migration: checkpoint: %w", err) + } + return nil +} diff --git a/pkg/registry/file/sqliteobject_migration_scale_test.go b/pkg/registry/file/sqliteobject_migration_scale_test.go new file mode 100644 index 000000000..d3ba55adf --- /dev/null +++ b/pkg/registry/file/sqliteobject_migration_scale_test.go @@ -0,0 +1,665 @@ +package file + +// The startup migration (§8.2) and the reverse export (§8.4) at a realistic +// scale: ~5,000 ContainerProfile rows of every reconcile shape in production +// proportions, seeded through the legacy StorageImpl over real files, run +// with the production batch size. The small-scale tests in +// sqliteobject_migration_test.go pin each shape's semantics; these pin that +// the counts, INV-2, idempotence, dry-run prediction, crash resumption and +// the export hold together on a corpus, and report the cost. + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "math/rand" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" + + helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/uuid" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitex" +) + +// scaleMix is the corpus shape: counts per reconcile outcome. The +// proportions are the assessment's (~95 % clean, ~2 % diverged, ~1 % each +// row/file-only, ~0.5 % each staging and orphan, a handful undecodable, a few +// wrong-type gobs through the external tool). +type scaleMix struct { + plain int // clean, non-TS + series int // clean TS series of two reports each + diverged int // row resourceVersion ahead of the file + rowOnly int // row without file + fileOnly int // file without row + stagingPairs int // objects named *.g.tail with staging siblings + plain staging files + orphans int // payloads rows without a metadata row + garbageRow int // undecodable gob, row present + garbageNoRow int // undecodable gob, no row (keeps the sweep armed) + toolOK int // wrong-type gob with a row, the external tool decodes it + toolNoRow int // wrong-type gob without a row, the tool decodes it (sweep) + toolFail int // wrong-type gob with a row, the tool fails +} + +func defaultScaleMix() scaleMix { + return scaleMix{ + plain: 4300, series: 260, + diverged: 100, rowOnly: 50, fileOnly: 50, + stagingPairs: 12, orphans: 25, + garbageRow: 5, garbageNoRow: 2, + toolOK: 4, toolNoRow: 1, toolFail: 2, + } +} + +// rows is the number of metadata rows the legacy binary leaves behind. +func (m scaleMix) rows() int { + return m.plain + 2*m.series + m.diverged + m.rowOnly + m.stagingPairs + m.garbageRow + m.toolOK + m.toolFail +} + +// expectedCounts is the report the migration must produce for the mix. +func (m scaleMix) expectedCounts() map[string]int { + return map[string]int{ + MigrationShapeMigrated: m.plain + 2*m.series + m.stagingPairs + m.toolOK, + MigrationShapeDiverged: m.diverged, + MigrationShapeRowWithoutFile: m.rowOnly, + MigrationShapeFileWithoutRow: m.fileOnly + m.toolNoRow, + MigrationShapeTempFile: 3 * m.stagingPairs, + MigrationShapeOrphanPayload: m.orphans, + MigrationShapeUndecodable + "/" + MigrationSourceFile: m.garbageRow + m.garbageNoRow + m.toolFail, + } +} + +// scaleCorpus is what the seeding left: every key with the object the legacy +// binary served for it (nil for keys that must not survive), plus the keys +// per shape for targeted assertions. +type scaleCorpus struct { + mix scaleMix + before map[string]*softwarecomposition.ContainerProfile + gone []string // rowOnly + orphans: no object after the migration + unchanged []string // garbageRow + toolFail: row left as is (rv NULL) + files int // payload files the legacy binary wrote +} + +// installSidecarMigrationTool points execMigrationTool at a script that +// answers from a JSON sidecar named after the payload file's basename, and +// fails when there is none: the tool-succeeds and tool-fails arms of +// decodeLegacyFile from one fixture. +func installSidecarMigrationTool(t *testing.T) (sidecarDir string) { + t.Helper() + sidecarDir = t.TempDir() + script := filepath.Join(sidecarDir, "fake-migration.sh") + body := "#!/bin/sh\n" + + "# args: -file -type \n" + + "f=\"$FAKE_SIDECAR_DIR/$(basename \"$2\").json\"\n" + + "if [ ! -f \"$f\" ]; then echo \"fake migration: no sidecar for $2\" >&2; exit 1; fi\n" + + "cat \"$f\"\n" + require.NoError(t, os.WriteFile(script, []byte(body), 0755)) + t.Setenv("FAKE_SIDECAR_DIR", sidecarDir) + old := migrationBinaryPath + migrationBinaryPath = script + t.Cleanup(func() { migrationBinaryPath = old }) + return sidecarDir +} + +// seedScaleCorpus writes the mix through the old binary and then applies +// each shape's damage (row edits through the fixture, file removals, +// staging and garbage files). Objects rotate through every template. +func (e *migrationEnv) seedScaleCorpus(mix scaleMix, sidecarDir string) *scaleCorpus { + t := e.t + t.Helper() + tpls := loadTemplates(t) // every testdata/p*.json (containerprofile_load_test.go) + c := &scaleCorpus{mix: mix, before: map[string]*softwarecomposition.ContainerProfile{}} + rng := rand.New(rand.NewSource(1)) + next := 0 + obj := func(name string, ts bool) *softwarecomposition.ContainerProfile { + p := tpls[next%len(tpls)].profile.DeepCopy() + next++ + p.Name = name + p.Namespace = e.ns + p.ResourceVersion = "" + p.UID = uuid.NewUUID() + if !ts { + delete(p.Annotations, helpersv1.ReportSeriesIdMetadataKey) + } + return p + } + create := func(p *softwarecomposition.ContainerProfile) string { + out := e.legacyCreate(p) + c.before[e.key(out.Name)] = out + c.files++ + return e.key(out.Name) + } + + for i := 0; i < mix.plain; i++ { + k := create(obj(fmt.Sprintf("plain-%05d", i), false)) + // A fifth of the clean rows sit at RV 2 or 3, as a live cluster's do. + if i%5 == 0 { + for n := 0; n < 1+i%2; n++ { + c.before[k] = e.legacyUpdate(k, func(cp *softwarecomposition.ContainerProfile) { + cp.Annotations["scale-test/updated"] = strconv.Itoa(n + 1) + }) + } + } + } + for i := 0; i < mix.series; i++ { + base := fmt.Sprintf("replicaset-scale-%04d-c-1111-2222", i) + for _, suffix := range []string{"0001", "0002"} { + create(obj(base+"-"+suffix, true)) + } + } + for i := 0; i < mix.diverged; i++ { + k := create(obj(fmt.Sprintf("diverged-%03d", i), false)) + setRowJSON(t, e.fixture, k, `$.resourceVersion`, strconv.Itoa(5+rng.Intn(20))) + } + for i := 0; i < mix.rowOnly; i++ { + k := create(obj(fmt.Sprintf("rowonly-%03d", i), false)) + require.NoError(t, e.fs.Remove(e.filePath(filepath.Base(k)))) + c.gone = append(c.gone, k) + delete(c.before, k) + c.files-- + } + for i := 0; i < mix.fileOnly; i++ { + k := create(obj(fmt.Sprintf("fileonly-%03d", i), false)) + if i%2 == 0 { + c.before[k] = e.legacyUpdate(k, func(cp *softwarecomposition.ContainerProfile) { cp.Annotations["scale-test/updated"] = "1" }) + } + _, _, kind, _, ns, name := K8sPathToKeys(k) + require.NoError(t, sqlitex.Execute(e.fixture, `DELETE FROM metadata WHERE kind = ? AND namespace = ? AND name = ?`, &sqlitex.ExecOptions{Args: []any{kind, ns, name}})) + } + dir := filepath.Dir(e.filePath("x")) + for i := 0; i < mix.stagingPairs; i++ { + // A workload legally named "*.g.tail": its payload file name contains + // ".g.t" and must survive; its staging siblings must not. + name := fmt.Sprintf("trap-%03d.g.tail", i) + create(obj(name, false)) + for _, n := range []string{name + ".g.t", fmt.Sprintf("%s.g.t.%d.%d", name, 1700000000000000000+i, i), fmt.Sprintf("staged-%03d.g.t", i)} { + require.NoError(t, afero.WriteFile(e.fs, filepath.Join(dir, n), []byte("staged"), 0644)) + } + } + for i := 0; i < mix.orphans; i++ { + name := fmt.Sprintf("orphan-%03d", i) + require.NoError(t, sqlitex.Execute(e.fixture, + `INSERT INTO payloads (kind, namespace, name, encoding, body) VALUES (?, ?, ?, ?, ?)`, + &sqlitex.ExecOptions{Args: []any{ContainerProfileKind, e.ns, name, PayloadEncodingJSONV1Beta1, []byte("{}")}})) + c.gone = append(c.gone, e.key(name)) + } + for i := 0; i < mix.garbageRow; i++ { + k := create(obj(fmt.Sprintf("garbage-%03d", i), false)) + require.NoError(t, afero.WriteFile(e.fs, e.filePath(filepath.Base(k)), []byte("not a gob stream"), 0644)) + c.unchanged = append(c.unchanged, k) + delete(c.before, k) + } + for i := 0; i < mix.garbageNoRow; i++ { + require.NoError(t, afero.WriteFile(e.fs, e.filePath(fmt.Sprintf("garbage-norow-%03d", i)), []byte("not a gob stream"), 0644)) + c.files++ + } + wrongType := gobPayloadNeedingMigration(t) + sidecar := func(k string) { + // What the real tool would print: the object as JSON. + b, err := json.Marshal(c.before[k]) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(sidecarDir, filepath.Base(k)+GobExt+".json"), b, 0644)) + } + for i := 0; i < mix.toolOK; i++ { + k := create(obj(fmt.Sprintf("toolok-%03d", i), false)) + sidecar(k) + require.NoError(t, afero.WriteFile(e.fs, e.filePath(filepath.Base(k)), wrongType, 0644)) + } + for i := 0; i < mix.toolNoRow; i++ { + k := create(obj(fmt.Sprintf("toolnorow-%03d", i), false)) + sidecar(k) + require.NoError(t, afero.WriteFile(e.fs, e.filePath(filepath.Base(k)), wrongType, 0644)) + _, _, kind, _, ns, name := K8sPathToKeys(k) + require.NoError(t, sqlitex.Execute(e.fixture, `DELETE FROM metadata WHERE kind = ? AND namespace = ? AND name = ?`, &sqlitex.ExecOptions{Args: []any{kind, ns, name}})) + } + for i := 0; i < mix.toolFail; i++ { + k := create(obj(fmt.Sprintf("toolfail-%03d", i), false)) + require.NoError(t, afero.WriteFile(e.fs, e.filePath(filepath.Base(k)), wrongType, 0644)) + c.unchanged = append(c.unchanged, k) + delete(c.before, k) + } + return c +} + +// assertScaleMigrated checks every surviving key against what the old +// binary served, INV-2 on every CP key, and the damaged keys' fates. +func (e *migrationEnv) assertScaleMigrated(c *scaleCorpus) { + t := e.t + t.Helper() + for k, legacyObj := range c.before { + got, err := e.storeGet(k) + require.NoError(t, err, k) + require.Equal(t, legacyObj.UID, got.UID, k) + want := legacyObj + if strings.Contains(k, "/diverged-") { + // The one intended difference: the row said 5..24, the file 1; + // the object is served at max+1 (the row JSON was rewritten to it). + require.Equal(t, strconv.FormatInt(parseRV(e.inspect(k).jsonRV), 10), got.ResourceVersion, k) + require.Greater(t, parseRV(got.ResourceVersion), int64(5), "%s: max(rowRV, payloadRV)+1", k) + want = legacyObj.DeepCopy() + want.ResourceVersion = got.ResourceVersion + } else { + require.Equal(t, legacyObj.ResourceVersion, got.ResourceVersion, k) + } + require.Equal(t, canonicalCP(want), canonicalCP(got), "the ObjectStore serves what the old binary served for %s", k) + } + for _, k := range c.gone { + row := e.inspect(k) + require.False(t, row.metaExists || row.payloadExists, "%s must not survive", k) + } + for _, k := range c.unchanged { + row := e.inspect(k) + require.True(t, row.metaExists && row.rv == nil && !row.payloadExists, "%s: an undecodable row is left exactly as it was", k) + exists, err := afero.Exists(e.fs, e.filePath(filepath.Base(k))) + require.NoError(t, err) + require.True(t, exists, "%s: an undecodable file is never deleted (PM-2)", k) + } + keys := allCPKeys(t, e.fixture) + require.Equal(t, len(c.before)+len(c.unchanged), len(keys), "exactly the surviving objects and the untouched undecodable rows remain") + for _, k := range keys { + if e.inspect(k).rv == nil { + continue // an unchanged undecodable row + } + assertINV2(t, e.fixture, k) + } +} + +// memReport is the memory cost of one phase: heap growth and the process's +// peak resident set (Linux VmHWM; 0 elsewhere). +type memReport struct { + heapAllocBefore, heapAllocAfter, totalAllocDelta, sysAfter, peakRSS uint64 +} + +func (m memReport) String() string { + return fmt.Sprintf("heapAlloc %dMB→%dMB totalAlloc +%dMB sys %dMB peakRSS %dMB", + m.heapAllocBefore>>20, m.heapAllocAfter>>20, m.totalAllocDelta>>20, m.sysAfter>>20, m.peakRSS>>20) +} + +func peakRSS() uint64 { + f, err := os.Open("/proc/self/status") + if err != nil { + return 0 + } + defer func() { _ = f.Close() }() + sc := bufio.NewScanner(f) + for sc.Scan() { + if strings.HasPrefix(sc.Text(), "VmHWM:") { + fields := strings.Fields(sc.Text()) + if len(fields) >= 2 { + kb, _ := strconv.ParseUint(fields[1], 10, 64) + return kb << 10 + } + } + } + return 0 +} + +func measureMem(fn func()) memReport { + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + fn() + runtime.ReadMemStats(&after) + return memReport{ + heapAllocBefore: before.HeapAlloc, heapAllocAfter: after.HeapAlloc, + totalAllocDelta: after.TotalAlloc - before.TotalAlloc, sysAfter: after.Sys, peakRSS: peakRSS(), + } +} + +// treeSize sums the sizes of every file under dir on the OS filesystem. +func treeSize(t *testing.T, dir string) (files int, bytes int64) { + t.Helper() + err := filepath.Walk(dir, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + files++ + bytes += info.Size() + } + return nil + }) + require.NoError(t, err) + return files, bytes +} + +func dbSize(t *testing.T, dbPath string) int64 { + t.Helper() + var total int64 + for _, p := range []string{dbPath, dbPath + "-wal"} { + if st, err := os.Stat(p); err == nil { + total += st.Size() + } + } + return total +} + +// scaleMigrationBound is the elapsed-time ceiling for the 5,000-row +// migration. Measured at ~1.4 s on a laptop (tmpfs); a 40× margin absorbs a +// slow CI runner while still failing a per-row full scan or a per-row +// checkpoint, either of which is minutes at this size. The race detector +// stretches the run ~20× (25 s measured), so its bound is four times wider. +func scaleMigrationBound() time.Duration { + if raceDetectorEnabled { + return 4 * time.Minute + } + return time.Minute +} + +// TestMigration_Scale_MixedCorpus: 5,000 rows of every shape, production +// batch size, exact counts, INV-2 everywhere, dry-run prediction, idempotent +// second run, elapsed under the bound, memory reported. +func TestMigration_Scale_MixedCorpus(t *testing.T) { + if testing.Short() { + t.Skip("scale test") + } + base := t.TempDir() + e := newMigrationEnv(t, afero.NewBasePathFs(afero.NewOsFs(), base)) + sidecars := installSidecarMigrationTool(t) + mix := defaultScaleMix() + seedStart := time.Now() + c := e.seedScaleCorpus(mix, sidecars) + t.Logf("seeded %d metadata rows, %d payload files, %d time_series rows in %s", mix.rows(), c.files, 2*mix.series, time.Since(seedStart)) + tsRowsBefore := countTimeSeriesRows(t, e.fixture) + require.GreaterOrEqual(t, tsRowsBefore, 500) + + // Dry-run first, with no gate (the flag-off operator check of §8.3). + dry, err := MigrateContainerProfiles(e.ctx, e.pool, nil, e.fs, DefaultStorageRoot, e.scheme, ContainerProfileMigrationOptions{DryRun: true}) + require.NoError(t, err) + require.Equal(t, mix.expectedCounts(), dry.Counts, "dry-run counts") + require.Nil(t, e.inspect(e.key("plain-00000")).rv, "dry-run wrote nothing") + + e.startNew() + var report *ContainerProfileMigrationReport + mem := measureMem(func() { + report = e.mustMigrate(ContainerProfileMigrationOptions{}) // production batch size + }) + t.Logf("migration: elapsed=%s batches=%d counts=%v mem: %s", report.Elapsed, report.Batches, report.Counts, mem) + require.Less(t, report.Elapsed, scaleMigrationBound()) + require.Equal(t, mix.expectedCounts(), report.Counts, "real-run counts") + require.Equal(t, dry.Counts, report.Counts, "the dry run predicted the real run exactly") + require.GreaterOrEqual(t, report.Batches, (mix.rows()+DefaultMigrationBatchSize-1)/DefaultMigrationBatchSize, "at least ceil(rows/%d) batches", DefaultMigrationBatchSize) + require.True(t, report.SweepsRun) + require.False(t, e.migrationDone(), "undecodable files without a row keep the sweep armed (PM-2)") + require.Equal(t, tsRowsBefore, countTimeSeriesRows(t, e.fixture), "time_series rows untouched") + + verifyStart := time.Now() + e.assertScaleMigrated(c) + t.Logf("verified %d surviving objects + INV-2 on every key in %s", len(c.before), time.Since(verifyStart)) + + // The migrated rows are live: a CAS update on a sample of every clean shape. + for _, name := range []string{"plain-00000", "plain-04299", "replicaset-scale-0000-c-1111-2222-0001", "diverged-000", "trap-000.g.tail", "toolok-000", "fileonly-001", "toolnorow-000"} { + require.NoError(t, e.storeUpdate(e.ctx, e.key(name)), name) + assertINV2(t, e.fixture, e.key(name)) + } + + // Second run: only the undecodable rows are met again; nothing is written. + again := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 0, again.Batches, "a second run commits nothing: %v", again.Counts) + for shape, n := range again.Counts { + if strings.HasPrefix(shape, MigrationShapeUndecodable) { + continue + } + require.Equal(t, 0, n, "second run reconciled %s", shape) + } + require.Equal(t, report.Count(MigrationShapeUndecodable), again.Count(MigrationShapeUndecodable), "undecodable payloads stay visible on every start") + t.Logf("second run: elapsed=%s counts=%v", again.Elapsed, again.Counts) +} + +func countTimeSeriesRows(t *testing.T, conn *sqlite.Conn) int { + t.Helper() + var n int + require.NoError(t, sqlitex.Execute(conn, `SELECT count(*) FROM time_series`, &sqlitex.ExecOptions{ResultFunc: func(stmt *sqlite.Stmt) error { n = int(stmt.ColumnInt64(0)); return nil }})) + return n +} + +// TestMigration_Scale_ResumesAfterProcessKill is TestMigration_ResumesAfter +// ProcessKill on the 5,000-row corpus: the child dies inside batch 40 of the +// production batch size, the parent resumes and completes. +func TestMigration_Scale_ResumesAfterProcessKill(t *testing.T) { + if testing.Short() { + t.Skip("scale test") + } + if os.Getenv("CP_MIGRATION_CRASH_DIR") != "" { + t.Skip("child helper only") + } + base := t.TempDir() + fs := afero.NewBasePathFs(afero.NewOsFs(), base) + e := newMigrationEnv(t, fs) + sidecars := installSidecarMigrationTool(t) + mix := defaultScaleMix() + c := e.seedScaleCorpus(mix, sidecars) + require.NoError(t, e.legacyPool.Close()) + require.NoError(t, e.pool.Close()) + e.pool, e.legacyPool = nil, nil + + const crashBatch = 40 + cmd := exec.Command(os.Args[0], "-test.run=^TestMigrationCrashChild$", "-test.v") + cmd.Env = append(os.Environ(), + "CP_MIGRATION_CRASH_DIR="+base, "CP_MIGRATION_CRASH_DB="+e.dbPath, "CP_MIGRATION_CRASH_TOOL="+migrationBinaryPath, + "CP_MIGRATION_CRASH_BATCH="+strconv.Itoa(crashBatch), "CP_MIGRATION_CRASH_BATCHSIZE="+strconv.Itoa(DefaultMigrationBatchSize)) + childStart := time.Now() + out, err := cmd.CombinedOutput() + var exitErr *exec.ExitError + require.True(t, errors.As(err, &exitErr), "the child must die: %v\n%s", err, out) + require.Equal(t, 3, exitErr.ExitCode(), "%s", out) + require.Contains(t, string(out), fmt.Sprintf("child: crashing inside batch %d", crashBatch)) + t.Logf("child ran %d batches and died in %s", crashBatch-1, time.Since(childStart)) + + pool := NewPoolWithOptions(e.dbPath, PoolOptions{Size: 4, BusyTimeout: 5 * time.Second, DisableAutoCheckpoint: true}) + armUngatedWriteCheck(t, pool) + e.pool = pool + e.fixture = openFixtureConn(t, pool, e.dbPath, 5*time.Second) + // Every candidate row is either fully reconciled or untouched; the rows + // the child's committed batches covered are exactly (crashBatch-1)*B + // candidates — some of which are row_without_file deletes (no row left) + // rather than rewrites, so count outcomes, not rewrites. + durable, pending := 0, 0 + for _, k := range allCPKeys(t, e.fixture) { + row := e.inspect(k) + if !row.metaExists { + continue + } + require.Equal(t, row.rv != nil, row.payloadExists, "no half-written key after the kill: %s", k) + if row.rv != nil { + durable++ + } else { + pending++ + } + } + require.Greater(t, durable, 0) + require.Greater(t, pending, 0) + t.Logf("after the kill: %d rows durable, %d pending", durable, pending) + + e.startNew() + report := e.mustMigrate(ContainerProfileMigrationOptions{}) + t.Logf("resumed: elapsed=%s batches=%d counts=%v", report.Elapsed, report.Batches, report.Counts) + require.Less(t, report.Elapsed, scaleMigrationBound()) + want := mix.expectedCounts() + // The child's committed batches covered the first (crashBatch-1)*B + // candidate rows in rowid order (clean plain rows: seeded first); the + // resume reconciles every other row candidate, and the sweeps (file + // shapes) run for the first time. + require.Equal(t, (crashBatch-1)*DefaultMigrationBatchSize, durable, "exactly the child's committed batches are durable") + rowCandidates := want[MigrationShapeMigrated] + want[MigrationShapeDiverged] + want[MigrationShapeRowWithoutFile] + resumedRows := report.Count(MigrationShapeMigrated) + report.Count(MigrationShapeDiverged) + report.Count(MigrationShapeRowWithoutFile) + require.Equal(t, rowCandidates-durable, resumedRows, "the resume reconciles exactly the rows the child had not committed: %v", report.Counts) + require.Equal(t, want[MigrationShapeFileWithoutRow], report.Count(MigrationShapeFileWithoutRow)) + require.Equal(t, want[MigrationShapeTempFile], report.Count(MigrationShapeTempFile)) + require.Equal(t, want[MigrationShapeOrphanPayload], report.Count(MigrationShapeOrphanPayload)) + e.assertScaleMigrated(c) + again := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 0, again.Batches, "%v", again.Counts) +} + +// ---- the export at scale, through the real cpexport binary ---- + +var ( + cpexportBinOnce sync.Once + cpexportBinPath string + cpexportBinErr error +) + +// buildCpexport compiles cmd/cpexport once per test binary. +func buildCpexport(t *testing.T) string { + t.Helper() + cpexportBinOnce.Do(func() { + dir, err := os.MkdirTemp("", "cpexport-bin") + if err != nil { + cpexportBinErr = err + return + } + bin := filepath.Join(dir, "cpexport") + cmd := exec.Command("go", "build", "-o", bin, "github.com/kubescape/storage/cmd/cpexport") + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + cpexportBinErr = fmt.Errorf("%w: %s", err, stderr.String()) + return + } + cpexportBinPath = bin + }) + require.NoError(t, cpexportBinErr, "go build cmd/cpexport") + return cpexportBinPath +} + +var cpexportReportRE = regexp.MustCompile(`cpexport: exported=(\d+) legacySkipped=(\d+) undecodable=(\d+) dryRun=(true|false) elapsed=(\S+)`) + +type cpexportResult struct { + exitCode int + exported, legacySkipped, undecodable int + dryRun bool + stdout, stderr string +} + +// runCpexport runs the built binary and parses its report line. +func runCpexport(t *testing.T, bin string, args ...string) cpexportResult { + t.Helper() + cmd := exec.Command(bin, args...) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + res := cpexportResult{stdout: stdout.String(), stderr: stderr.String()} + var exitErr *exec.ExitError + switch { + case err == nil: + case errors.As(err, &exitErr): + res.exitCode = exitErr.ExitCode() + default: + t.Fatalf("run cpexport: %v", err) + } + if m := cpexportReportRE.FindStringSubmatch(res.stdout); m != nil { + res.exported, _ = strconv.Atoi(m[1]) + res.legacySkipped, _ = strconv.Atoi(m[2]) + res.undecodable, _ = strconv.Atoi(m[3]) + res.dryRun = m[4] == "true" + } + return res +} + +// TestExport_Scale_BinaryAgainstMigratedCorpus: migrate the 5,000-row corpus, +// then run the real cpexport binary against the root (dry-run, then for +// real), and prove every exported file decodes to what the ObjectStore +// serves. Files and payload rows coexist during the rollback window; the +// disk overhead of that window is reported. +func TestExport_Scale_BinaryAgainstMigratedCorpus(t *testing.T) { + if testing.Short() { + t.Skip("scale test") + } + bin := buildCpexport(t) + base := t.TempDir() + e := newMigrationEnv(t, afero.NewBasePathFs(afero.NewOsFs(), base)) + sidecars := installSidecarMigrationTool(t) + mix := defaultScaleMix() + c := e.seedScaleCorpus(mix, sidecars) + e.startNew() + report := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, mix.expectedCounts(), report.Counts) + // The store keeps writing after the flip: keys the old binary has no + // file for (created) or a stale file for (updated) are what the export + // exists for. + created := 200 + for i := 0; i < created; i++ { + e.storeCreate(fmt.Sprintf("created-under-the-new-store-%03d", i)) + } + updated := 300 + for i := 0; i < updated; i++ { + require.NoError(t, e.storeUpdate(e.ctx, e.key(fmt.Sprintf("plain-%05d", i)))) + } + // The server is stopped for the export (the binary's contract). + e.stopNew() + + root := filepath.Join(base, DefaultStorageRoot) + filesBefore, bytesBefore := treeSize(t, root) + dbBefore := dbSize(t, e.dbPath) + // Every migrated row plus the store's own writes. The rows the migration + // left rv NULL have no payloads row and are outside the export's join + // entirely (their file is the only copy and stays); LegacySkipped counts + // only rv NULL rows WITH a payloads row (legacy_rewrite), none here. + wantExported := len(c.before) + created + wantSkipped := 0 + + dry := runCpexport(t, bin, "-root", root, "-db", e.dbPath, "-dry-run") + require.Equal(t, 0, dry.exitCode, "stderr: %s", dry.stderr) + require.True(t, dry.dryRun) + require.Equal(t, wantExported, dry.exported, dry.stdout) + require.Equal(t, wantSkipped, dry.legacySkipped, dry.stdout) + require.Equal(t, 0, dry.undecodable, dry.stdout) + filesAfterDry, _ := treeSize(t, root) + require.Equal(t, filesBefore, filesAfterDry, "dry-run wrote nothing") + + start := time.Now() + real := runCpexport(t, bin, "-root", root, "-db", e.dbPath) + elapsed := time.Since(start) + require.Equal(t, 0, real.exitCode, "stderr: %s", real.stderr) + require.False(t, real.dryRun) + require.Equal(t, dry.exported, real.exported, "the dry run predicted the export") + require.Equal(t, dry.legacySkipped, real.legacySkipped) + require.Equal(t, 0, real.undecodable) + filesAfter, bytesAfter := treeSize(t, root) + require.Equal(t, filesBefore+created, filesAfter, "exactly the store-created keys gained a file; no staging file left behind") + t.Logf("export: %d files in %s (binary wall); files %d→%d (%dMB→%dMB), db+wal %dMB→%dMB; during the rollback window the %dMB of files duplicate the rows' payloads (total on disk %dMB)", + real.exported, elapsed, filesBefore, filesAfter, bytesBefore>>20, bytesAfter>>20, dbBefore>>20, dbSize(t, e.dbPath)>>20, bytesAfter>>20, (bytesAfter+dbSize(t, e.dbPath))>>20) + + // Every exported file decodes to what the ObjectStore serves, at the + // row's resourceVersion and UID, and the old binary reads it. + e.startNew() + verifyStart := time.Now() + decoded := 0 + for k := range c.before { + want := e.mustStoreGet(k) + got := decodeGob(t, e.readFile(filepath.Base(k))) + require.Equal(t, canonicalCP(want), canonicalCP(got), "%s: the exported file is not what the store serves", k) + require.Equal(t, want.ResourceVersion, got.ResourceVersion, k) + require.Equal(t, want.UID, got.UID, k) + decoded++ + } + for i := 0; i < created; i++ { + k := e.key(fmt.Sprintf("created-under-the-new-store-%03d", i)) + want := e.mustStoreGet(k) + got := decodeGob(t, e.readFile(filepath.Base(k))) + require.Equal(t, canonicalCP(want), canonicalCP(got), k) + decoded++ + } + require.Equal(t, real.exported, decoded, "every exported file was decoded") + e.stopNew() + // The old binary serves a sample at the post-flip state. + for _, name := range []string{"plain-00000", "plain-00299", "created-under-the-new-store-199", "diverged-042", "trap-003.g.tail"} { + got := e.legacyGet(e.key(name)) + require.Equal(t, name, got.Name) + } + t.Logf("verified %d exported files in %s", decoded, time.Since(verifyStart)) +} diff --git a/pkg/registry/file/sqliteobject_migration_test.go b/pkg/registry/file/sqliteobject_migration_test.go new file mode 100644 index 000000000..c851eeef6 --- /dev/null +++ b/pkg/registry/file/sqliteobject_migration_test.go @@ -0,0 +1,940 @@ +package file + +// Tests of the startup ContainerProfile data migration (§8.2, K-1, K-2, R3, +// PM-2, the §9 "integration additions from Part C"). +// +// The "old binary" is a real legacy StorageImpl (no guard, no gate) over a +// second pool on the same database file and the same afero.Fs: it writes the +// exact row + gob file a pre-flag process writes, and the AC-G1 ledger is +// vacuous for its pool. The "new binary" is the gated pool (AC-G1 armed), +// the migration and an ObjectStore built on it; startNew/stopNew are one boot. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "sync/atomic" + "testing" + "time" + + helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/kubescape/storage/pkg/apis/softwarecomposition/install" + "github.com/kubescape/storage/pkg/config" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apiserver/pkg/storage" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitemigration" + "zombiezen.com/go/sqlite/sqlitex" +) + +type migrationEnv struct { + t *testing.T + ctx context.Context + dir string + dbPath string + fs afero.Fs + scheme *runtime.Scheme + // pool is the new binary's pool: AC-G1 armed for the whole test. + pool *sqlitemigration.Pool + // legacyPool/legacy are the old binary: an unguarded, ungated + // StorageImpl with the CP processor, writing rows and gob files. + legacyPool *sqlitemigration.Pool + legacy *StorageImpl + fixture *sqlite.Conn + tpl softwarecomposition.ContainerProfile + ns string + + gate *writeGate + store *ObjectStore +} + +// newMigrationEnv builds the environment over fs (a MemMapFs unless the +// test needs real files for a child process). +func newMigrationEnv(t *testing.T, fs afero.Fs) *migrationEnv { + t.Helper() + dir := t.TempDir() + dbPath := filepath.Join(dir, "metadata.sq3") + pool := NewPoolWithOptions(dbPath, PoolOptions{Size: 4, BusyTimeout: 5 * time.Second, DisableAutoCheckpoint: true}) + armUngatedWriteCheck(t, pool) + sch := runtime.NewScheme() + install.Install(sch) + // The old binary: SQLite defaults (autocheckpoint on), no gate, no guard. + legacyPool := NewPoolWithOptions(dbPath, PoolOptions{Size: 4, BusyTimeout: 5 * time.Second}) + legacyProcessor := NewContainerProfileProcessor(config.Config{DefaultNamespace: "kubescape", MaxContainerProfileSize: 40000}, nil) + legacyProcessor.Interval = 0 + legacy := NewStorageImplWithCollector(fs, DefaultStorageRoot, legacyPool, NewWatchDispatcher(), sch, legacyProcessor).(*StorageImpl) + + content, err := os.ReadFile("testdata/p1.json") + require.NoError(t, err) + var tpl softwarecomposition.ContainerProfile + require.NoError(t, json.Unmarshal(content, &tpl)) + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + t.Cleanup(cancel) + e := &migrationEnv{ + t: t, ctx: ctx, dir: dir, dbPath: dbPath, fs: fs, scheme: sch, + pool: pool, legacyPool: legacyPool, legacy: legacy, + tpl: tpl, ns: tpl.Namespace, + } + e.fixture = openFixtureConn(t, pool, dbPath, 5*time.Second) + t.Cleanup(func() { + e.stopNew() + for _, p := range []*sqlitemigration.Pool{e.pool, e.legacyPool} { + if p == nil { + continue + } + closed := make(chan error, 1) + go func() { closed <- p.Close() }() + select { + case err := <-closed: + require.NoError(t, err) + case <-time.After(10 * time.Second): + t.Errorf("pool.Close did not return") + } + } + }) + return e +} + +func (e *migrationEnv) key(name string) string { return testCPPrefix + e.ns + "/" + name } + +func (e *migrationEnv) filePath(name string) string { + return filepath.Join(DefaultStorageRoot, e.key(name)) + GobExt +} + +// plain returns a non-TS profile named name, with the UID the REST layer +// assigns before storage sees an object. +func (e *migrationEnv) plain(name string) *softwarecomposition.ContainerProfile { + p := e.tpl.DeepCopy() + p.Name = name + p.ResourceVersion = "" + p.UID = uuid.NewUUID() + delete(p.Annotations, helpersv1.ReportSeriesIdMetadataKey) + return p +} + +// ts returns a TS profile of series suffix under base name base. +func (e *migrationEnv) ts(base, suffix string) *softwarecomposition.ContainerProfile { + p := e.tpl.DeepCopy() + p.Name = base + "-" + suffix + p.ResourceVersion = "" + p.UID = uuid.NewUUID() + return p +} + +// legacyCreate writes p through the old binary: row (rv NULL) + gob file. +func (e *migrationEnv) legacyCreate(p *softwarecomposition.ContainerProfile) *softwarecomposition.ContainerProfile { + e.t.Helper() + out := &softwarecomposition.ContainerProfile{} + require.NoError(e.t, e.legacy.Create(e.ctx, e.key(p.Name), p, out, 0)) + return out +} + +// legacyUpdate mutates key through the old binary's GuaranteedUpdate: the +// file is rewritten at RV+1, then the row is INSERT OR REPLACEd (rv NULL). +func (e *migrationEnv) legacyUpdate(key string, mutate func(*softwarecomposition.ContainerProfile)) *softwarecomposition.ContainerProfile { + e.t.Helper() + out := &softwarecomposition.ContainerProfile{} + require.NoError(e.t, e.legacy.GuaranteedUpdate(e.ctx, key, out, false, nil, + func(input runtime.Object, _ storage.ResponseMeta) (runtime.Object, *uint64, error) { + cp := input.(*softwarecomposition.ContainerProfile).DeepCopy() + mutate(cp) + return cp, nil, nil + }, nil)) + return out +} + +func (e *migrationEnv) legacyGet(key string) *softwarecomposition.ContainerProfile { + e.t.Helper() + out := &softwarecomposition.ContainerProfile{} + require.NoError(e.t, e.legacy.Get(e.ctx, key, storage.GetOptions{}, out)) + return out +} + +// startNew boots the new binary: the gate and an ObjectStore on the armed pool. +func (e *migrationEnv) startNew() { + e.t.Helper() + require.Nil(e.t, e.gate, "startNew called twice") + gateCtx, cancel := context.WithTimeout(e.ctx, 30*time.Second) + defer cancel() + gate, err := newWriteGate(gateCtx, e.pool) + require.NoError(e.t, err) + e.gate = gate + processor := NewContainerProfileProcessor(config.Config{DefaultNamespace: "kubescape", MaxContainerProfileSize: 40000}, nil) + processor.Interval = 0 + guarded := NewStorageImpl(afero.NewMemMapFs(), DefaultStorageRoot, e.pool, nil, e.scheme).(*StorageImpl) + guarded.SetForeignKinds(IsContainerProfileKind) + guarded.SetWriteGate(gate) + store, err := NewObjectStore(e.pool, e.dbPath, nil, e.scheme, processor, guarded, gate, ObjectStoreOptions{CheckpointInterval: time.Hour}) + require.NoError(e.t, err) + e.store = store +} + +// stopNew shuts the new binary down (a rollback, or a crash-free restart). +func (e *migrationEnv) stopNew() { + e.t.Helper() + if e.store != nil { + require.NoError(e.t, e.store.Close()) + e.store = nil + } + if e.gate != nil { + require.NoError(e.t, e.gate.Close()) + e.gate = nil + } +} + +func (e *migrationEnv) migrate(opts ContainerProfileMigrationOptions) (*ContainerProfileMigrationReport, error) { + e.t.Helper() + return MigrateContainerProfiles(e.ctx, e.pool, e.gate, e.fs, DefaultStorageRoot, e.scheme, opts) +} + +func (e *migrationEnv) mustMigrate(opts ContainerProfileMigrationOptions) *ContainerProfileMigrationReport { + e.t.Helper() + report, err := e.migrate(opts) + require.NoError(e.t, err) + return report +} + +func (e *migrationEnv) storeGet(key string) (*softwarecomposition.ContainerProfile, error) { + out := &softwarecomposition.ContainerProfile{} + err := e.store.Get(e.ctx, key, storage.GetOptions{}, out) + return out, err +} + +func (e *migrationEnv) mustStoreGet(key string) *softwarecomposition.ContainerProfile { + e.t.Helper() + out, err := e.storeGet(key) + require.NoError(e.t, err) + return out +} + +// storeUpdate bumps an annotation through the ObjectStore's CAS with a +// bounded ctx: a row the CAS can never match (rv NULL) times out. +func (e *migrationEnv) storeUpdate(ctx context.Context, key string) error { + out := &softwarecomposition.ContainerProfile{} + return e.store.GuaranteedUpdate(ctx, key, out, false, nil, + func(input runtime.Object, _ storage.ResponseMeta) (runtime.Object, *uint64, error) { + cp := input.(*softwarecomposition.ContainerProfile).DeepCopy() + cp.Annotations["migration-test/touched"] = strconv.FormatInt(time.Now().UnixNano(), 10) + return cp, nil, nil + }, nil) +} + +func (e *migrationEnv) inspect(key string) dbRow { + e.t.Helper() + return inspectRow(e.t, e.fixture, key) +} + +func (e *migrationEnv) assertINV2(keys ...string) { + e.t.Helper() + for _, k := range keys { + assertINV2(e.t, e.fixture, k) + } +} + +func (e *migrationEnv) migrationDone() bool { + e.t.Helper() + var state string + require.NoError(e.t, sqlitex.Execute(e.fixture, `SELECT state FROM migration_state WHERE name = ?`, &sqlitex.ExecOptions{ + Args: []any{migrationStateName}, ResultFunc: func(stmt *sqlite.Stmt) error { state = stmt.ColumnText(0); return nil }, + })) + return state == migrationStateDone +} + +// seed writes n plain profiles and one two-report TS series through the old +// binary and returns every key with the object the old binary served for it. +func (e *migrationEnv) seed(n int) map[string]*softwarecomposition.ContainerProfile { + e.t.Helper() + before := map[string]*softwarecomposition.ContainerProfile{} + for i := 0; i < n; i++ { + p := e.legacyCreate(e.plain(fmt.Sprintf("plain-%02d", i))) + before[e.key(p.Name)] = p + } + base := "replicaset-x-y-1111-2222" + for _, suffix := range []string{"0001", "0002"} { + p := e.legacyCreate(e.ts(base, suffix)) + before[e.key(p.Name)] = p + } + // One key at RV 2, so the migration meets a non-initial version. + first := e.key("plain-00") + before[first] = e.legacyUpdate(first, func(cp *softwarecomposition.ContainerProfile) { + cp.Annotations["migration-test/updated"] = "1" + }) + for k := range before { + before[k] = e.legacyGet(k) + } + return before +} + +func TestMigration_LegacyRowsBecomeObjectStoreRows(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + before := e.seed(5) + for k := range before { + row := e.inspect(k) + require.True(t, row.metaExists) + require.Nil(t, row.rv, "a legacy row has rv NULL") + require.False(t, row.payloadExists, "a legacy row has no payloads row") + } + tsRowsBefore := e.inspect(e.key("replicaset-x-y-1111-2222")).tsRows + require.Equal(t, 2, tsRowsBefore, "the old binary wrote the series' time_series rows") + + e.startNew() + report := e.mustMigrate(ContainerProfileMigrationOptions{BatchSize: 3}) + require.Equal(t, len(before), report.Count(MigrationShapeMigrated), "%v", report.Counts) + require.Equal(t, len(before), report.Work()) + require.Equal(t, 4, report.Batches, "7 rows in batches of 3, plus the done-flag write: %v", report.Counts) + require.True(t, report.SweepsRun) + require.True(t, e.migrationDone()) + + keys := make([]string, 0, len(before)) + for k, legacyObj := range before { + keys = append(keys, k) + got := e.mustStoreGet(k) + require.Equal(t, canonicalCP(legacyObj), canonicalCP(got), "the ObjectStore serves what the old binary served for %s", k) + require.Equal(t, legacyObj.ResourceVersion, got.ResourceVersion, "resourceVersion preserved for %s", k) + require.Equal(t, legacyObj.UID, got.UID) + row := e.inspect(k) + require.Equal(t, legacyObj.ResourceVersion, strconv.FormatInt(*row.rv, 10)) + exists, err := afero.Exists(e.fs, filepath.Join(DefaultStorageRoot, k)+GobExt) + require.NoError(t, err) + require.True(t, exists, "legacy files are left in place for the export tool (§8.4)") + } + e.assertINV2(keys...) + require.Equal(t, tsRowsBefore, e.inspect(e.key("replicaset-x-y-1111-2222")).tsRows, "time_series rows untouched") + + // The migrated rows are live: CAS updates and a fresh Create work. + for _, k := range keys { + require.NoError(t, e.storeUpdate(e.ctx, k), "CAS update of migrated key %s", k) + got := e.mustStoreGet(k) + require.Equal(t, strconv.FormatInt(parseRV(before[k].ResourceVersion)+1, 10), got.ResourceVersion) + } + created := &softwarecomposition.ContainerProfile{} + require.NoError(t, e.store.Create(e.ctx, e.key("fresh"), e.plain("fresh"), created, 0)) + e.assertINV2(append(keys, e.key("fresh"))...) + + // Idempotent: the second start reconciles nothing and skips the sweeps. + again := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 0, again.Work(), "%v", again.Counts) + require.Equal(t, 0, again.Batches) + require.False(t, again.SweepsRun, "the done-flag gates the file sweeps") +} + +func TestMigration_ReconcileShapes(t *testing.T) { + t.Run("diverged: row ahead of file gets max+1", func(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + p := e.legacyCreate(e.plain("diverged")) + key := e.key(p.Name) + // B16's shape: the row says 5, the file says 1. + setRowJSON(t, e.fixture, key, `$.resourceVersion`, "5") + e.startNew() + report := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 1, report.Count(MigrationShapeDiverged), "%v", report.Counts) + require.Equal(t, 0, report.Count(MigrationShapeMigrated)) + got := e.mustStoreGet(key) + require.Equal(t, "6", got.ResourceVersion, "max(rowRV, payloadRV)+1") + e.assertINV2(key) + require.NoError(t, e.storeUpdate(e.ctx, key)) + require.Equal(t, "7", e.mustStoreGet(key).ResourceVersion) + }) + + t.Run("completed row beats learning file (PreSave's non-TS revert)", func(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + p := e.legacyCreate(e.plain("revert")) + key := e.key(p.Name) + require.Equal(t, helpersv1.Learning, p.Annotations[helpersv1.StatusMetadataKey]) + setRowJSON(t, e.fixture, key, fmt.Sprintf(`$.annotations."%s"`, helpersv1.StatusMetadataKey), helpersv1.Completed) + e.startNew() + report := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 1, report.Count(MigrationShapeMigrated), "%v", report.Counts) + require.Equal(t, helpersv1.Completed, e.mustStoreGet(key).Annotations[helpersv1.StatusMetadataKey]) + e.assertINV2(key) + }) + + t.Run("row without file is deleted", func(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + p := e.legacyCreate(e.plain("nofile")) + key := e.key(p.Name) + require.NoError(t, e.fs.Remove(e.filePath(p.Name))) + e.startNew() + report := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 1, report.Count(MigrationShapeRowWithoutFile), "%v", report.Counts) + row := e.inspect(key) + require.False(t, row.metaExists) + require.False(t, row.payloadExists) + _, err := e.storeGet(key) + require.True(t, storage.IsNotFound(err)) + }) + + t.Run("file without row is imported by the sweep", func(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + p := e.legacyCreate(e.plain("norow")) + key := e.key(p.Name) + e.legacyUpdate(key, func(cp *softwarecomposition.ContainerProfile) { cp.Annotations["x"] = "y" }) + fileObj := e.legacyGet(key) + require.Equal(t, "2", fileObj.ResourceVersion) + _, _, kind, _, ns, name := K8sPathToKeys(key) + require.NoError(t, sqlitex.Execute(e.fixture, `DELETE FROM metadata WHERE kind = ? AND namespace = ? AND name = ?`, &sqlitex.ExecOptions{Args: []any{kind, ns, name}})) + e.startNew() + report := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 1, report.Count(MigrationShapeFileWithoutRow), "%v", report.Counts) + got := e.mustStoreGet(key) + require.Equal(t, canonicalCP(fileObj), canonicalCP(got)) + require.Equal(t, "2", got.ResourceVersion, "imported at the file's resourceVersion") + require.Equal(t, fileObj.UID, got.UID, "imported at the file's UID") + e.assertINV2(key) + require.True(t, e.migrationDone()) + }) + + t.Run("staging files are removed, payloads whose name contains .g.t are not", func(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + e.legacyCreate(e.plain("keep")) + // A workload named "keep.g.tail" (a legal DNS subdomain) has a payload + // file whose name contains ".g.t"; and an undecodable one of the same + // shape must survive as the only copy of its object (PM-2). + e.legacyCreate(e.plain("keep.g.tail")) + require.NoError(t, afero.WriteFile(e.fs, e.filePath("garbage.g.tail"), []byte("not a gob stream"), 0644)) + require.NoError(t, sqlitex.Execute(e.fixture, + `INSERT INTO metadata (kind, namespace, name, metadata) VALUES (?, ?, ?, ?)`, + &sqlitex.ExecOptions{Args: []any{ContainerProfileKind, e.ns, "garbage.g.tail", `{"name":"garbage.g.tail","namespace":"` + e.ns + `","resourceVersion":"1"}`}})) + dir := filepath.Dir(e.filePath("keep")) + for _, n := range []string{"a.g.t", "b.g.t.1234.5", "keep.g.tail.g.t", "keep.g.tail.g.t.99.1"} { + require.NoError(t, afero.WriteFile(e.fs, filepath.Join(dir, n), []byte("staged"), 0644)) + } + e.startNew() + report := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 4, report.Count(MigrationShapeTempFile), "%v", report.Counts) + require.Equal(t, 2, report.Count(MigrationShapeMigrated), "%v", report.Counts) + require.Equal(t, 1, report.Count(MigrationShapeUndecodable), "%v", report.Counts) + for _, n := range []string{"a.g.t", "b.g.t.1234.5", "keep.g.tail.g.t", "keep.g.tail.g.t.99.1"} { + exists, err := afero.Exists(e.fs, filepath.Join(dir, n)) + require.NoError(t, err) + require.False(t, exists) + } + for _, n := range []string{"keep", "keep.g.tail", "garbage.g.tail"} { + exists, err := afero.Exists(e.fs, e.filePath(n)) + require.NoError(t, err) + require.True(t, exists, "%s is a payload file, not a staging file", n) + } + e.assertINV2(e.key("keep.g.tail")) + require.Equal(t, "keep.g.tail", e.mustStoreGet(e.key("keep.g.tail")).Name) + row := e.inspect(e.key("garbage.g.tail")) + require.True(t, row.metaExists, "the undecodable object's row is left as is") + require.Nil(t, row.rv) + // The next start meets the same undecodable row again, never a row_without_file. + again := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 1, again.Count(MigrationShapeUndecodable), "%v", again.Counts) + require.Equal(t, 0, again.Count(MigrationShapeRowWithoutFile), "%v", again.Counts) + }) + + t.Run("undecodable payloads are skipped and counted, never deleted", func(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + p := e.legacyCreate(e.plain("garbage")) + key := e.key(p.Name) + require.NoError(t, afero.WriteFile(e.fs, e.filePath(p.Name), []byte("not a gob stream"), 0644)) + // A garbage file with no row at all. + require.NoError(t, afero.WriteFile(e.fs, e.filePath("garbage-norow"), []byte("not a gob stream"), 0644)) + e.startNew() + report := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 2, report.Count(MigrationShapeUndecodable), "%v", report.Counts) + require.Equal(t, 0, report.Count(MigrationShapeRowWithoutFile)) + row := e.inspect(key) + require.True(t, row.metaExists, "the row is left as is") + require.Nil(t, row.rv) + require.False(t, row.payloadExists) + for _, n := range []string{"garbage", "garbage-norow"} { + exists, err := afero.Exists(e.fs, e.filePath(n)) + require.NoError(t, err) + require.True(t, exists, "an undecodable file is never deleted (PM-2)") + } + require.False(t, e.migrationDone(), "an undecodable file without a row keeps the sweep armed for the next start") + again := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.True(t, again.SweepsRun) + require.Equal(t, 2, again.Count(MigrationShapeUndecodable), "still reported on every start: %v", again.Counts) + }) +} + +// R3 / K-1: after a rollback, the old binary's GuaranteedUpdate rewrites the +// .g file at RV+1 and INSERT OR REPLACEs the row (rv NULL). Before the +// reconcile the ObjectStore serves the stale payloads body and can never +// update the key; the next start repairs it from the FILE. +func TestMigration_R3_LegacyRewriteRepairsFromTheFile(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + before := e.seed(2) + e.startNew() + e.mustMigrate(ContainerProfileMigrationOptions{}) + key := e.key("plain-01") + e.assertINV2(key) + migrated := e.mustStoreGet(key) + require.Equal(t, "1", migrated.ResourceVersion) + + // Rollback: the old binary runs and updates the key. + e.stopNew() + e.legacyUpdate(key, func(cp *softwarecomposition.ContainerProfile) { + cp.Annotations["migration-test/legacy-write"] = "after-rollback" + }) + legacyView := e.legacyGet(key) + require.Equal(t, "2", legacyView.ResourceVersion) + row := e.inspect(key) + require.Nil(t, row.rv, "INSERT OR REPLACE nulled rv") + require.Nil(t, row.uid) + require.True(t, row.payloadExists, "the payloads row (now stale) survived the legacy write") + + // Re-enable WITHOUT the reconcile: the shape's two symptoms. + e.startNew() + stale := e.mustStoreGet(key) + require.Equal(t, "1", stale.ResourceVersion, "the payloads body is the stale side") + require.Empty(t, stale.Annotations["migration-test/legacy-write"], "the legacy write is invisible through the stale body") + shortCtx, cancel := context.WithTimeout(e.ctx, 700*time.Millisecond) + err := e.storeUpdate(shortCtx, key) + cancel() + require.Error(t, err, "rv NULL never matches the CAS: every update conflicts until the ctx expires") + + // The every-start reconcile repairs it from the file. + report := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 1, report.Counts[MigrationShapeLegacyRewrite+"/"+MigrationSourceFile], "%v", report.Counts) + require.Equal(t, 0, report.Counts[MigrationShapeLegacyRewrite+"/"+MigrationSourcePayloads]) + require.False(t, report.SweepsRun) + repaired := e.mustStoreGet(key) + require.Equal(t, canonicalCP(legacyView), canonicalCP(repaired), "the body after repair is the FILE's content, not the pre-rollback payload") + require.Equal(t, "after-rollback", repaired.Annotations["migration-test/legacy-write"]) + require.Equal(t, "2", repaired.ResourceVersion, "rv := the JSON's resourceVersion, no +1") + row = e.inspect(key) + require.Equal(t, "2", row.jsonRV) + e.assertINV2(key) + require.NoError(t, e.storeUpdate(e.ctx, key), "the next CAS succeeds") + require.Equal(t, "3", e.mustStoreGet(key).ResourceVersion) + // The other key was never touched by the old binary and is not rewritten. + other := e.key("plain-00") + e.assertINV2(other) + require.Equal(t, before[other].ResourceVersion, e.mustStoreGet(other).ResourceVersion) +} + +// K-1's row-only variant: a legacy row write with no file (readMetadata's +// sidecar branch) keeps the payloads body as the source. +func TestMigration_R3_LegacyRewriteRowOnlyKeepsThePayloadsBody(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + e.seed(1) + key := e.key("plain-00") + e.startNew() + e.mustMigrate(ContainerProfileMigrationOptions{}) + migrated := e.mustStoreGet(key) + require.Equal(t, "2", migrated.ResourceVersion) + e.stopNew() + + require.NoError(t, e.fs.Remove(e.filePath("plain-00"))) + _, _, kind, _, ns, name := K8sPathToKeys(key) + require.NoError(t, sqlitex.Execute(e.fixture, + `INSERT OR REPLACE INTO metadata (kind, namespace, name, metadata) SELECT kind, namespace, name, metadata FROM metadata WHERE kind = ? AND namespace = ? AND name = ?`, + &sqlitex.ExecOptions{Args: []any{kind, ns, name}})) + row := e.inspect(key) + require.Nil(t, row.rv) + require.True(t, row.payloadExists) + + e.startNew() + report := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 1, report.Counts[MigrationShapeLegacyRewrite+"/"+MigrationSourcePayloads], "%v", report.Counts) + repaired := e.mustStoreGet(key) + require.Equal(t, canonicalCP(migrated), canonicalCP(repaired), "the payloads body is kept") + require.Equal(t, "2", repaired.ResourceVersion) + e.assertINV2(key) + require.NoError(t, e.storeUpdate(e.ctx, key)) +} + +// legacy_rewrite from a stale file behind a newer payloads body (no +// export before the rollback): the content is the file's (K-1) but the +// version never regresses below what the body — and any watcher — saw. +func TestMigration_R3_LegacyRewriteNeverRegressesBelowTheBodyVersion(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + e.seed(1) + key := e.key("plain-00") + e.startNew() + e.mustMigrate(ContainerProfileMigrationOptions{}) + require.NoError(t, e.storeUpdate(e.ctx, key)) + require.Equal(t, "3", e.mustStoreGet(key).ResourceVersion, "the body is at 3; the pre-migration file stays at 2") + e.stopNew() + // The old binary rewrites the row from its stale file's view: JSON at 2, rv NULL. + _, _, kind, _, ns, name := K8sPathToKeys(key) + require.NoError(t, sqlitex.Execute(e.fixture, + `INSERT OR REPLACE INTO metadata (kind, namespace, name, metadata) SELECT kind, namespace, name, json_set(CAST(metadata AS TEXT), '$.resourceVersion', '2') FROM metadata WHERE kind = ? AND namespace = ? AND name = ?`, + &sqlitex.ExecOptions{Args: []any{kind, ns, name}})) + require.Nil(t, e.inspect(key).rv) + + e.startNew() + report := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 1, report.Counts[MigrationShapeLegacyRewrite+"/"+MigrationSourceFile], "%v", report.Counts) + got := e.mustStoreGet(key) + require.Equal(t, "3", got.ResourceVersion, "max(json 2, file 2, body 3)") + require.Empty(t, got.Annotations["migration-test/touched"], "the content is the legacy writer's file (K-1)") + e.assertINV2(key) + require.NoError(t, e.storeUpdate(e.ctx, key)) + require.Equal(t, "4", e.mustStoreGet(key).ResourceVersion) +} + +// K-2 (fixed by rollback-safety-guard Part 1): a legacy delete after a +// rollback used to remove the row and the file but leave the payloads row +// behind, so every Create of the key failed on the UNIQUE constraint until +// the mirror predicate deleted the orphan. deleteLocked's non-gated arm now +// deletes the payloads row itself (before the metadata row, for crash +// safety), so no orphan is created and Create succeeds immediately. +func TestMigration_K2_OrphanPayloadAfterLegacyDelete(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + e.seed(2) + key := e.key("plain-01") + e.startNew() + e.mustMigrate(ContainerProfileMigrationOptions{}) + e.stopNew() + + require.NoError(t, e.legacy.Delete(e.ctx, key, &softwarecomposition.ContainerProfile{}, nil, nil, nil, storage.DeleteOptions{})) + row := e.inspect(key) + require.False(t, row.metaExists) + require.False(t, row.payloadExists, "Part 1: the old binary's delete now cleans up payloads too, so no orphan is left") + exists, err := afero.Exists(e.fs, e.filePath("plain-01")) + require.NoError(t, err) + require.False(t, exists) + + e.startNew() + created := &softwarecomposition.ContainerProfile{} + require.NoError(t, e.store.Create(e.ctx, key, e.plain("plain-01"), created, 0), "Create succeeds immediately: there is no orphan payloads row to conflict with") + + report := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 0, report.Count(MigrationShapeOrphanPayload), "%v", report.Counts) + e.assertINV2(key, e.key("plain-00")) +} + +// A crash in the middle of a batch rolls that batch back; the next start +// completes the migration from the predicate with nothing lost or duplicated. +func TestMigration_ResumesAfterCrashMidBatch(t *testing.T) { + for _, mode := range []string{"error", "panic"} { + t.Run(mode, func(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + before := e.seed(10) // 12 rows: batches of 5 → 5, 5, 2 + e.startNew() + var crashed bool + opts := ContainerProfileMigrationOptions{BatchSize: 5} + opts.hooks.beforeStatement = func(batch, idx int, name string) error { + if batch == 2 && idx == 3 && !crashed { + crashed = true + if mode == "panic" { + panic(errInjectedCrash) + } + return errInjectedCrash + } + return nil + } + report, err := e.migrate(opts) + require.Error(t, err) + require.True(t, crashed) + require.Contains(t, err.Error(), "batch 2 rolled back") + require.Equal(t, 2, report.Batches) + + // Exactly the first batch is durable; the second rolled back whole. + var done, pending int + for k := range before { + row := e.inspect(k) + require.True(t, row.metaExists, "no row is lost by a crash") + if row.rv != nil { + require.True(t, row.payloadExists, "a committed row has its payload") + done++ + } else { + require.False(t, row.payloadExists, "a rolled-back row has no half-written payload") + pending++ + } + } + require.Equal(t, 5, done) + require.Equal(t, 7, pending) + require.False(t, e.migrationDone()) + + // Restart. + e.stopNew() + e.startNew() + report = e.mustMigrate(ContainerProfileMigrationOptions{BatchSize: 5}) + require.Equal(t, 7, report.Count(MigrationShapeMigrated), "only the pending rows are reconciled: %v", report.Counts) + require.Equal(t, 0, report.Count(MigrationShapeDiverged)) + require.True(t, e.migrationDone()) + keys := make([]string, 0, len(before)) + for k, legacyObj := range before { + keys = append(keys, k) + got := e.mustStoreGet(k) + require.Equal(t, canonicalCP(legacyObj), canonicalCP(got)) + require.Equal(t, legacyObj.ResourceVersion, got.ResourceVersion) + } + e.assertINV2(keys...) + require.Equal(t, 0, e.mustMigrate(ContainerProfileMigrationOptions{}).Work()) + }) + } +} + +// The re-exec variant: the child process is killed (os.Exit inside the gated +// transaction) with real files on disk; the parent resumes. +func TestMigration_ResumesAfterProcessKill(t *testing.T) { + if os.Getenv("CP_MIGRATION_CRASH_DIR") != "" { + t.Skip("child helper only") + } + base := t.TempDir() + fs := afero.NewBasePathFs(afero.NewOsFs(), base) + e := newMigrationEnv(t, fs) + before := e.seed(10) + // Hand the database to the child: close our pools' hold on it first. + require.NoError(t, e.legacyPool.Close()) + require.NoError(t, e.pool.Close()) + e.pool, e.legacyPool = nil, nil + + cmd := exec.Command(os.Args[0], "-test.run=^TestMigrationCrashChild$", "-test.v") + cmd.Env = append(os.Environ(), "CP_MIGRATION_CRASH_DIR="+base, "CP_MIGRATION_CRASH_DB="+e.dbPath) + out, err := cmd.CombinedOutput() + var exitErr *exec.ExitError + require.True(t, errors.As(err, &exitErr), "the child must die: %v\n%s", err, out) + require.Equal(t, 3, exitErr.ExitCode(), "the child exits from inside the batch\n%s", out) + require.Contains(t, string(out), "child: crashing inside batch 2") + + // The parent boots a new binary on the same files and database. + pool := NewPoolWithOptions(e.dbPath, PoolOptions{Size: 4, BusyTimeout: 5 * time.Second, DisableAutoCheckpoint: true}) + armUngatedWriteCheck(t, pool) + gate, err := newWriteGate(e.ctx, pool) + require.NoError(t, err) + processor := NewContainerProfileProcessor(config.Config{DefaultNamespace: "kubescape", MaxContainerProfileSize: 40000}, nil) + processor.Interval = 0 + guarded := NewStorageImpl(afero.NewMemMapFs(), DefaultStorageRoot, pool, nil, e.scheme).(*StorageImpl) + guarded.SetForeignKinds(IsContainerProfileKind) + guarded.SetWriteGate(gate) + store, err := NewObjectStore(pool, e.dbPath, nil, e.scheme, processor, guarded, gate, ObjectStoreOptions{CheckpointInterval: time.Hour}) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, store.Close()) + require.NoError(t, gate.Close()) + require.NoError(t, pool.Close()) + }) + fixture := openFixtureConn(t, pool, e.dbPath, 5*time.Second) + + done := 0 + for k := range before { + row := inspectRow(t, fixture, k) + require.True(t, row.metaExists) + require.Equal(t, row.rv != nil, row.payloadExists, "no half-written key after the kill") + if row.rv != nil { + done++ + } + } + require.Equal(t, 5, done, "exactly the child's first batch is durable") + + report, err := MigrateContainerProfiles(e.ctx, pool, gate, fs, DefaultStorageRoot, e.scheme, ContainerProfileMigrationOptions{BatchSize: 5}) + require.NoError(t, err) + require.Equal(t, 7, report.Count(MigrationShapeMigrated), "%v", report.Counts) + for k, legacyObj := range before { + assertINV2(t, fixture, k) + got := &softwarecomposition.ContainerProfile{} + require.NoError(t, store.Get(e.ctx, k, storage.GetOptions{}, got)) + require.Equal(t, canonicalCP(legacyObj), canonicalCP(got)) + require.Equal(t, legacyObj.ResourceVersion, got.ResourceVersion) + } +} + +// TestMigrationCrashChild is the child of TestMigration_ResumesAfterProcessKill +// (and its scale variant): it runs the migration on the parent's files and +// exits from inside the transaction of batch CP_MIGRATION_CRASH_BATCH +// (default 2) at batch size CP_MIGRATION_CRASH_BATCHSIZE (default 5). +func TestMigrationCrashChild(t *testing.T) { + base := os.Getenv("CP_MIGRATION_CRASH_DIR") + if base == "" { + t.Skip("child helper only") + } + dbPath := os.Getenv("CP_MIGRATION_CRASH_DB") + if tool := os.Getenv("CP_MIGRATION_CRASH_TOOL"); tool != "" { + migrationBinaryPath = tool + } + crashBatch, batchSize := 2, 5 + if v, err := strconv.Atoi(os.Getenv("CP_MIGRATION_CRASH_BATCH")); err == nil { + crashBatch = v + } + if v, err := strconv.Atoi(os.Getenv("CP_MIGRATION_CRASH_BATCHSIZE")); err == nil { + batchSize = v + } + fs := afero.NewBasePathFs(afero.NewOsFs(), base) + pool := NewPoolWithOptions(dbPath, PoolOptions{Size: 4, BusyTimeout: 5 * time.Second, DisableAutoCheckpoint: true}) + sch := runtime.NewScheme() + install.Install(sch) + gate, err := newWriteGate(context.Background(), pool) + require.NoError(t, err) + opts := ContainerProfileMigrationOptions{BatchSize: batchSize} + opts.hooks.beforeStatement = func(batch, idx int, name string) error { + if batch == crashBatch && idx == 3 { + fmt.Printf("child: crashing inside batch %d before %s\n", batch, name) + os.Exit(3) + } + return nil + } + _, err = MigrateContainerProfiles(context.Background(), pool, gate, fs, DefaultStorageRoot, sch, opts) + require.NoError(t, err) + t.Fatal("the child must not reach the end of the migration") +} + +// Dry-run reconciles and counts the same shapes and writes nothing. +func TestMigration_DryRunWritesNothing(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + before := e.seed(3) + // An orphan payloads row and a staging file, seeded through the fixture. + require.NoError(t, sqlitex.Execute(e.fixture, + `INSERT INTO payloads (kind, namespace, name, encoding, body) VALUES (?, ?, ?, ?, ?)`, + &sqlitex.ExecOptions{Args: []any{ContainerProfileKind, e.ns, "orphan", PayloadEncodingJSONV1Beta1, []byte("{}")}})) + staged := filepath.Join(filepath.Dir(e.filePath("plain-00")), "staged.g.t") + require.NoError(t, afero.WriteFile(e.fs, staged, []byte("staged"), 0644)) + + // No gate exists in a flag-off dry run. + dry, err := MigrateContainerProfiles(e.ctx, e.pool, nil, e.fs, DefaultStorageRoot, e.scheme, ContainerProfileMigrationOptions{DryRun: true}) + require.NoError(t, err) + require.True(t, dry.DryRun) + require.Equal(t, len(before), dry.Count(MigrationShapeMigrated), "%v", dry.Counts) + require.Equal(t, 1, dry.Count(MigrationShapeOrphanPayload)) + require.Equal(t, 1, dry.Count(MigrationShapeTempFile)) + require.True(t, dry.SweepsRun) + for k := range before { + row := e.inspect(k) + require.Nil(t, row.rv, "dry-run wrote nothing") + require.False(t, row.payloadExists) + } + require.True(t, e.inspect(e.key("orphan")).payloadExists) + exists, err := afero.Exists(e.fs, staged) + require.NoError(t, err) + require.True(t, exists) + require.False(t, e.migrationDone()) + + _, err = MigrateContainerProfiles(e.ctx, e.pool, nil, e.fs, DefaultStorageRoot, e.scheme, ContainerProfileMigrationOptions{}) + require.Error(t, err, "a real run needs the gate") + + e.startNew() + real := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, dry.Counts, real.Counts, "the dry run predicted the real run") + require.True(t, e.migrationDone()) +} + +// The migration's batches are gated writes: with the gate held by another +// writer the migration waits its turn rather than busy-waiting on SQLite's +// lock (AC-G1 is armed on the env's pool and would fail an ungated batch). +func TestMigration_BatchesAreGatedWrites(t *testing.T) { + e := newMigrationEnv(t, afero.NewMemMapFs()) + e.seed(2) + e.startNew() + holderIn := make(chan struct{}) + release := make(chan struct{}) + holderDone := make(chan error, 1) + go func() { + holderDone <- e.gate.run(e.ctx, priorityHigh, "test-holder", "test", func(_ context.Context, _ *sqlite.Conn) error { + close(holderIn) + <-release + return nil + }) + }() + <-holderIn + migrated := make(chan *ContainerProfileMigrationReport, 1) + go func() { migrated <- e.mustMigrate(ContainerProfileMigrationOptions{}) }() + select { + case <-migrated: + t.Fatal("the migration committed while another writer held the gate") + case <-time.After(300 * time.Millisecond): + } + close(release) + require.NoError(t, <-holderDone) + select { + case report := <-migrated: + require.Equal(t, 4, report.Count(MigrationShapeMigrated), "%v", report.Counts) + case <-time.After(10 * time.Second): + t.Fatal("the migration did not proceed after the gate was released") + } +} + +// setRowJSON sets one json path of key's metadata JSON through the fixture +// (a B16-style edit of the row only). +func setRowJSON(t *testing.T, conn *sqlite.Conn, key, path, value string) { + t.Helper() + _, _, kind, _, ns, name := K8sPathToKeys(key) + require.NoError(t, sqlitex.Execute(conn, + `UPDATE metadata SET metadata = json_set(CAST(metadata AS TEXT), ?, ?) WHERE kind = ? AND namespace = ? AND name = ?`, + &sqlitex.ExecOptions{Args: []any{path, value, kind, ns, name}})) + require.Equal(t, int64(1), int64(conn.Changes())) +} + +// toggleFailFs fails Stat for one target path while armed, otherwise +// delegates. afero.Walk falls back to Stat (not Lstat) for entries when the +// underlying Fs is not an Lstater, which afero.NewMemMapFs() is not, so +// arming this on a file inside a walked directory reproduces a per-entry +// stat error mid-walk (a real filesystem's unreadable subtree/permission +// error) without needing an actual restricted filesystem. +type toggleFailFs struct { + afero.Fs + target string + fail atomic.Bool +} + +func (f *toggleFailFs) Stat(name string) (os.FileInfo, error) { + if f.fail.Load() && name == f.target { + return nil, fmt.Errorf("toggleFailFs: simulated stat failure for %s", name) + } + return f.Fs.Stat(name) +} + +// TestMigration_SweepWalkErrorDoesNotMarkDone: a filesystem error hit while +// walking one entry (a permission-denied subtree, in production) used to be +// swallowed by the walk callback's `if err != nil { return nil }`, so the +// sweep silently skipped whatever orphan payload sat under that entry and +// still reported success -- MigrateContainerProfiles then persisted the +// done marker over an incomplete sweep, with nothing left to trigger a +// retry on a later restart. The walk error must instead fail the migration, +// and a later restart (once the filesystem error clears) must complete the +// sweep it never got to. +func TestMigration_SweepWalkErrorDoesNotMarkDone(t *testing.T) { + failing := &toggleFailFs{Fs: afero.NewMemMapFs()} + e := newMigrationEnv(t, failing) + p := e.legacyCreate(e.plain("norow")) + key := e.key(p.Name) + failing.target = e.filePath(p.Name) + _, _, kind, _, ns, name := K8sPathToKeys(key) + require.NoError(t, sqlitex.Execute(e.fixture, `DELETE FROM metadata WHERE kind = ? AND namespace = ? AND name = ?`, + &sqlitex.ExecOptions{Args: []any{kind, ns, name}})) + e.startNew() + + failing.fail.Store(true) + _, err := e.migrate(ContainerProfileMigrationOptions{}) + require.Error(t, err, "a walk error must fail the migration, not silently succeed") + require.False(t, e.migrationDone(), "the done marker must not persist over an incomplete sweep") + + failing.fail.Store(false) + report := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 1, report.Count(MigrationShapeFileWithoutRow), "%v", + "the retried sweep on restart imports the orphan it never reached before") + require.True(t, e.migrationDone()) +} + +// TestMigration_SweepTopLevelStatErrorDoesNotMarkDone: the same class of bug +// at the top-level directory check -- DirExists returns exists=false on ANY +// stat error, not just "does not exist", and the caller reads a nil error +// here as "nothing to sweep, mark done". +func TestMigration_SweepTopLevelStatErrorDoesNotMarkDone(t *testing.T) { + failing := &toggleFailFs{Fs: afero.NewMemMapFs()} + e := newMigrationEnv(t, failing) + p := e.legacyCreate(e.plain("norow")) + key := e.key(p.Name) + failing.target = filepath.Join(DefaultStorageRoot, softwarecomposition.GroupName, ContainerProfileKind) + _, _, kind, _, ns, name := K8sPathToKeys(key) + require.NoError(t, sqlitex.Execute(e.fixture, `DELETE FROM metadata WHERE kind = ? AND namespace = ? AND name = ?`, + &sqlitex.ExecOptions{Args: []any{kind, ns, name}})) + e.startNew() + + failing.fail.Store(true) + _, err := e.migrate(ContainerProfileMigrationOptions{}) + require.Error(t, err, "a top-level stat error must fail the migration, not read as \"directory absent\"") + require.False(t, e.migrationDone()) + + failing.fail.Store(false) + report := e.mustMigrate(ContainerProfileMigrationOptions{}) + require.Equal(t, 1, report.Count(MigrationShapeFileWithoutRow)) + require.True(t, e.migrationDone()) +} diff --git a/pkg/registry/file/sqliteobject_store.go b/pkg/registry/file/sqliteobject_store.go new file mode 100644 index 000000000..02473aa7c --- /dev/null +++ b/pkg/registry/file/sqliteobject_store.go @@ -0,0 +1,1085 @@ +package file + +// ObjectStore: the SQLite-native, fully-ACID storage.Interface for the +// ContainerProfile kind (design: .omc/plans/full-acid-storage-architecture.md, +// §3). The object's metadata row (with real rv/uid columns), its payload BLOB +// (versioned JSON) and its time_series row are written in ONE BEGIN IMMEDIATE … +// COMMIT on the gate's dedicated connection; the compare-and-swap is one UPDATE +// … WHERE rv=:rv AND uid=:uid. Nothing but SQL on already-prepared bytes runs +// while the gate is held (INV-1): no lock, no callback, no dispatch, no +// decode, no encode. +// +// The startup data migration is sqliteobject_migration.go (§8); the cleanup +// handler's CP arm and GeneratedNetworkPolicyStorage read through this store +// (§5.6); the 13 legacy kinds share the gate (write-gate-sharing.md). Working +// package name in the design is sqliteobject; it lives in package file so the +// in-package measurement harness (containerprofile_load_test.go) can select it. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "time" + + "github.com/kubescape/go-logger" + "github.com/kubescape/go-logger/helpers" + helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + "github.com/kubescape/storage/pkg/metrics" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/conversion" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/apiserver/pkg/storage" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitemigration" + "zombiezen.com/go/sqlite/sqlitex" +) + +// PayloadEncodingJSONV1Beta1 is the payloads.encoding value for a body that is +// the object converted to the v1beta1 storage version and JSON-marshalled. +const PayloadEncodingJSONV1Beta1 = "json/v1beta1" + +// Write-path labels for storage_sqlite_write_hold_seconds: the ObjectStore's +// own, then the legacy kinds' sites routed through the shared gate +// (.omc/plans/write-gate-sharing.md §3.2). +const ( + holdPathCreate = "create" + holdPathUpdate = "update" + holdPathDelete = "delete" + holdPathConsolidate = "consolidate" + holdPathTimeSeries = "time_series" + + holdPathLegacyCommit = "legacy_commit" // W1/W2: the shard commit and saveObject + holdPathLegacyDelete = "legacy_delete" // W3: deleteLocked + holdPathRepair = "repair" // W4/W5/W6a/W7a: get()'s self-repair deletes + holdPathMigrate = "migrate" // W6b/W7b/W8: the gob-migration rewrite + holdPathCleanup = "cleanup" // W9a: the cleanup tick's row delete + holdPathCleanupMigrate = "cleanup_migrate" // W9b: the cleanup tick's sidecar-file migration + holdPathCPMigration = "cp_migration" // §8.2: one startup data-migration batch +) + +// ObjectStoreOptions tunes an ObjectStore. +type ObjectStoreOptions struct { + // CheckpointThresholdBytes is the -wal size that triggers a background + // checkpoint after a gated commit; non-positive = DefaultCheckpointThresholdBytes. + CheckpointThresholdBytes int64 + // CheckpointInterval is the timer fallback; non-positive = DefaultCheckpointInterval. + CheckpointInterval time.Duration +} + +// objectStoreHooks are test seams. Every field is nil in production. +type objectStoreHooks struct { + // beforeStatement runs inside the gated transaction before staged + // statement idx of a write set (crash-injection tests return an error or + // panic from it, or roll the transaction back underneath the holder). + beforeStatement func(conn *sqlite.Conn, path string, idx int, name string) error + // afterPrepare runs after a write's prepare phase, before its gate ticket. + afterPrepare func(path, key string) + // onPoolTake runs on every pool connection acquisition by the store or its + // ContainerProfileStorage (INV-1's "no pool wait inside the hold" probe). + onPoolTake func() +} + +// ObjectStore implements storage.Interface over the metadata + payloads + +// time_series tables. See the file comment. +type ObjectStore struct { + pool *sqlitemigration.Pool + scheme *runtime.Scheme + versioner storage.Versioner + processor Processor + watchDispatcher eventDispatcher + gate *writeGate + checkpointer *checkpointer + reservations *keyReservations + hooks objectStoreHooks +} + +// eventDispatcher is the slice of WatchDispatcher the store uses; an interface +// so INV-1's test can wrap it and prove no dispatch happens under the gate. +type eventDispatcher interface { + Added(key string, metaOut, obj runtime.Object) + Modified(key string, metaOut, obj runtime.Object) + Deleted(key string, metaOut runtime.Object) +} + +var _ storage.Interface = (*ObjectStore)(nil) + +// NewObjectStore builds the store over pool (the same pool/database file the +// legacy StorageImpl uses) and gate (the process's one write gate, shared +// with the legacy StorageImpl and the cleanup handler; the store neither +// builds nor closes it), starts the checkpointer and hands the processor its +// ContainerProfileStorage. dbPath is the database file (for the -wal size +// check); sbomStore is the legacy StorageImpl through which GetSbom reads the +// sbomsyft kind (R7). +func NewObjectStore(pool *sqlitemigration.Pool, dbPath string, watchDispatcher *WatchDispatcher, scheme *runtime.Scheme, processor Processor, sbomStore storage.Interface, gate *WriteGate, opts ObjectStoreOptions) (*ObjectStore, error) { + if watchDispatcher == nil { + watchDispatcher = NewWatchDispatcher() + } + if processor == nil { + processor = DefaultProcessor{} + } + if gate == nil { + return nil, errors.New("ObjectStore: a write gate is required") + } + if gate.pool != pool { + return nil, errors.New("ObjectStore: the write gate belongs to another pool") + } + s := &ObjectStore{ + pool: pool, + scheme: scheme, + versioner: storage.APIObjectVersioner{}, + processor: processor, + watchDispatcher: watchDispatcher, + gate: gate, + checkpointer: newCheckpointer(pool, dbPath, opts.CheckpointThresholdBytes, opts.CheckpointInterval), + reservations: newKeyReservations(), + } + s.checkpointer.start() + processor.SetStorage(newObjectStoreCPStorage(s, sbomStore)) + return s, nil +} + +// Close stops the checkpointer. The shared gate is closed by its owner +// (main.go's pre-shutdown hook), after every store's Close and before +// Pool.Close (K-5). +func (s *ObjectStore) Close() error { + s.checkpointer.Stop() + return nil +} + +// ---- storage.Interface plumbing ---- + +func (s *ObjectStore) Versioner() storage.Versioner { return s.versioner } +func (s *ObjectStore) ReadinessCheck() error { return nil } +func (s *ObjectStore) RequestWatchProgress(context.Context) error { return nil } +func (s *ObjectStore) GetCurrentResourceVersion(context.Context) (uint64, error) { return 0, nil } +func (s *ObjectStore) EnableResourceSizeEstimation(storage.KeysFunc) error { return nil } +func (s *ObjectStore) CompactRevision() int64 { return 0 } +func (s *ObjectStore) SetKeysFunc(storage.KeysFunc) {} +func (s *ObjectStore) Stats(context.Context) (storage.Stats, error) { + return storage.Stats{}, fmt.Errorf("unimplemented") +} + +func (s *ObjectStore) Watch(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) { + nw := newWatcher(ctx, opts.ResourceVersion == softwarecomposition.ResourceVersionFullSpec) + if wd, ok := s.watchDispatcher.(*WatchDispatcher); ok { + wd.Register(key, nw) + } else if r, ok := s.watchDispatcher.(interface{ Register(string, *watcher) }); ok { + r.Register(key, nw) + } + return nw, nil +} + +// takeConn takes a pool connection for a read or prepare phase. +func (s *ObjectStore) takeConn(ctx context.Context, op, key string) (*sqlite.Conn, error) { + if s.hooks.onPoolTake != nil { + s.hooks.onPoolTake() + } + poolCtx, cancel := poolContext() + defer cancel() + before := time.Now() + conn, err := s.pool.Take(poolCtx) + if err != nil { + metrics.ObservePoolWait(resourceFromKey(key), metrics.OutcomeTimeout, time.Since(before)) + return nil, newContentionTimeoutError(op, key, err) + } + metrics.ObservePoolWait(resourceFromKey(key), metrics.OutcomeAcquired, time.Since(before)) + conn.SetInterrupt(ctx.Done()) + return conn, nil +} + +// ---- codec ---- + +// encodeBody converts obj to the v1beta1 storage version and marshals it. The +// original TypeMeta is kept (ConvertToVersion stamps the target GVK), so a +// decode returns the object with exactly the apiVersion/kind it was saved with. +func (s *ObjectStore) encodeBody(obj runtime.Object) ([]byte, error) { + return encodePayloadBody(s.scheme, obj) +} + +// decodeBody unmarshals a payloads.body into objPtr through the v1beta1 type. +func (s *ObjectStore) decodeBody(encoding string, body []byte, objPtr runtime.Object) error { + return decodePayloadBody(s.scheme, encoding, body, objPtr) +} + +// encodePayloadBody is encodeBody without the receiver, shared with the +// startup data migration. +func encodePayloadBody(scheme *runtime.Scheme, obj runtime.Object) ([]byte, error) { + versioned, err := scheme.ConvertToVersion(obj, v1beta1.SchemeGroupVersion) + if err != nil { + return nil, fmt.Errorf("convert to v1beta1: %w", err) + } + versioned.GetObjectKind().SetGroupVersionKind(obj.GetObjectKind().GroupVersionKind()) + body, err := json.Marshal(versioned) + if err != nil { + return nil, fmt.Errorf("marshal payload: %w", err) + } + return body, nil +} + +// decodePayloadBody is decodeBody without the receiver. +func decodePayloadBody(scheme *runtime.Scheme, encoding string, body []byte, objPtr runtime.Object) error { + if encoding != PayloadEncodingJSONV1Beta1 { + return fmt.Errorf("unsupported payload encoding %q", encoding) + } + gvks, _, err := scheme.ObjectKinds(objPtr) + if err != nil || len(gvks) == 0 { + return fmt.Errorf("object kinds: %w", err) + } + versioned, err := scheme.New(v1beta1.SchemeGroupVersion.WithKind(gvks[0].Kind)) + if err != nil { + return fmt.Errorf("new v1beta1 %s: %w", gvks[0].Kind, err) + } + if err := json.Unmarshal(body, versioned); err != nil { + return fmt.Errorf("unmarshal payload: %w", err) + } + if err := scheme.Convert(versioned, objPtr, nil); err != nil { + return fmt.Errorf("convert from v1beta1: %w", err) + } + objPtr.GetObjectKind().SetGroupVersionKind(versioned.GetObjectKind().GroupVersionKind()) + return nil +} + +// calculateChecksum is StorageImpl.CalculateChecksum without the receiver. +func (s *ObjectStore) calculateChecksum(in runtime.Object) (string, error) { + return (&StorageImpl{scheme: s.scheme}).CalculateChecksum(in) +} + +// ---- rows ---- + +// objRow is one (metadata JOIN payloads) row. +type objRow struct { + metadataJSON []byte + rv int64 + uid string + encoding string + body []byte +} + +func (s *ObjectStore) readRow(conn *sqlite.Conn, key string) (*objRow, error) { + _, _, kind, _, namespace, name := K8sPathToKeys(key) + var row *objRow + err := sqlitex.Execute(conn, + `SELECT m.metadata, m.rv, m.uid, p.encoding, p.body + FROM metadata m JOIN payloads p USING (kind, namespace, name) + WHERE m.kind = :kind AND m.namespace = :namespace AND m.name = :name`, + &sqlitex.ExecOptions{ + Named: map[string]any{":kind": kind, ":namespace": namespace, ":name": name}, + ResultFunc: func(stmt *sqlite.Stmt) error { + r := &objRow{ + metadataJSON: []byte(stmt.ColumnText(0)), + rv: stmt.ColumnInt64(1), + uid: stmt.ColumnText(2), + encoding: stmt.ColumnText(3), + } + r.body = make([]byte, stmt.ColumnLen(4)) + stmt.ColumnBytes(4, r.body) + row = r + return nil + }, + }) + if err != nil { + return nil, fmt.Errorf("read object row: %w", err) + } + return row, nil +} + +// getWithConn is Get on a given connection. The metadata variant reads the +// shared metadata row exactly as the legacy store does; the full variant is +// one autocommit SELECT over the join (a WAL snapshot; no per-key lock). +func (s *ObjectStore) getWithConn(ctx context.Context, conn *sqlite.Conn, key string, opts storage.GetOptions, objPtr runtime.Object) (*objRow, error) { + if opts.ResourceVersion == softwarecomposition.ResourceVersionMetadata { + metadata, err := ReadMetadata(conn, key) + if err != nil { + if errors.Is(err, ErrMetadataNotFound) { + if opts.IgnoreNotFound { + return nil, runtime.SetZeroValue(objPtr) + } + return nil, storage.NewKeyNotFoundError(key, 0) + } + return nil, fmt.Errorf("read metadata: %w", err) + } + return nil, json.Unmarshal(metadata, objPtr) + } + row, err := s.readRow(conn, key) + if err != nil { + return nil, err + } + if row == nil { + if opts.IgnoreNotFound { + return nil, runtime.SetZeroValue(objPtr) + } + return nil, storage.NewKeyNotFoundError(key, 0) + } + if err := s.decodeBody(row.encoding, row.body, objPtr); err != nil { + logger.L().Ctx(ctx).Error("ObjectStore.Get - decode payload failed", helpers.Error(err), helpers.String("key", key)) + return nil, err + } + return row, nil +} + +// Get implements storage.Interface. +func (s *ObjectStore) Get(ctx context.Context, key string, opts storage.GetOptions, objPtr runtime.Object) error { + conn, err := s.takeConn(ctx, "get", key) + if err != nil { + return err + } + defer s.pool.Put(conn) + _, err = s.getWithConn(ctx, conn, key, opts, objPtr) + return err +} + +// GetList implements storage.Interface. The metadata variant is byte-identical +// to the legacy store's (same listMetadata statement, same continue token = +// rowid); the fullSpec variant is a paginated join, one statement per page. +func (s *ObjectStore) GetList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error { + ctx, predicate, v, elem, limit, batchSize, cursor, isFullSpec, err := (&StorageImpl{}).prepareGetList(ctx, key, opts, listObj) + if err != nil { + return err + } + pageLast := "" + for limit == 0 || int64(v.Len()) < limit { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + remaining := nextPageSize(limit, batchSize, int64(v.Len())) + conn, err := s.takeConn(ctx, "list", key) + if err != nil { + return err + } + fetched, err := s.fetchListPage(ctx, conn, key, cursor, remaining, isFullSpec, predicate, v, elem) + s.pool.Put(conn) + if err != nil { + return err + } + pageLast = fetched.pageLast + if int64(fetched.count) < remaining { + pageLast = "" + break + } + cursor = pageLast + } + return setListContinue(listObj, pageLast) +} + +func (s *ObjectStore) fetchListPage(ctx context.Context, conn *sqlite.Conn, key, cursor string, remaining int64, isFullSpec bool, predicate storage.SelectionPredicate, v reflect.Value, elem reflect.Type) (listPageResult, error) { + var objs []runtime.Object + var pageLast string + var count int + if !isFullSpec { + entries, last, err := listMetadata(conn, key, cursor, remaining) + if err != nil { + return listPageResult{}, fmt.Errorf("list objects for %q: %w", key, err) + } + pageLast, count = last, len(entries) + for _, entry := range entries { + obj := reflect.New(elem).Interface().(runtime.Object) + if err := json.Unmarshal([]byte(entry), obj); err != nil { + logger.L().Ctx(ctx).Error("ObjectStore.GetList - unmarshal metadata failed", helpers.Error(err), helpers.String("key", key)) + continue + } + objs = append(objs, obj) + } + } else { + _, _, kind, _, namespace, _ := K8sPathToKeys(key) + if cursor == "" { + cursor = "0" + } + type page struct { + encoding string + body []byte + } + var rows []page + // The page is selected on metadata by rowid FIRST (the legacy + // listMetadataKeys statement), then joined to payloads by primary key: + // design PM-1's "apply the predicate before the join". A plain join + // let SQLite drive from payloads and touch every body (Tier B: LIST + // p95 6.8 -> 80 ms). + // + // LEFT JOIN, not JOIN: count/pageLast below must reflect how many + // METADATA rows this page scanned (bounded by :limit), not how many + // of them had a payload row. A migration can leave a metadata row + // with no payload (an undecodable legacy file, left for the export + // tool); an inner join silently drops that row from the result, so + // count would undercount the page relative to :limit even when many + // more valid rows exist beyond it -- GetList's "count < remaining -> + // EOF" check would then stop pagination early and hide every row + // after the gap. Rows with no payload are logged and skipped from + // objs below, exactly as an undecodable one already is, but still + // counted and still advance pageLast. + err := sqlitex.Execute(conn, + `SELECT m.rowid, p.encoding, p.body, p.body IS NULL + FROM (SELECT rowid, kind, namespace, name FROM metadata + WHERE kind = :kind + AND (:namespace = '' OR namespace = :namespace) + AND rowid > :cont + AND is_time_series = 0 + ORDER BY rowid + LIMIT :limit) m + LEFT JOIN payloads p ON p.kind = m.kind AND p.namespace = m.namespace AND p.name = m.name + ORDER BY m.rowid`, + &sqlitex.ExecOptions{ + Named: map[string]any{":kind": kind, ":namespace": namespace, ":cont": cursor, ":limit": remaining}, + ResultFunc: func(stmt *sqlite.Stmt) error { + pageLast = stmt.ColumnText(0) + if stmt.ColumnInt64(3) == 1 { + logger.L().Ctx(ctx).Warning("ObjectStore.GetList - metadata row has no payload row; skipped", helpers.String("key", key)) + rows = append(rows, page{}) + return nil + } + p := page{encoding: stmt.ColumnText(1), body: make([]byte, stmt.ColumnLen(2))} + stmt.ColumnBytes(2, p.body) + rows = append(rows, p) + return nil + }, + }) + if err != nil { + return listPageResult{}, fmt.Errorf("list objects for %q: %w", key, err) + } + count = len(rows) + for _, r := range rows { + if r.encoding == "" && r.body == nil { + continue + } + obj := reflect.New(elem).Interface().(runtime.Object) + if err := s.decodeBody(r.encoding, r.body, obj); err != nil { + logger.L().Ctx(ctx).Error("ObjectStore.GetList - decode payload failed", helpers.Error(err), helpers.String("key", key)) + continue + } + objs = append(objs, obj) + } + } + v.Grow(len(objs)) + for _, obj := range objs { + matched, err := predicate.Matches(obj) + if err != nil { + return listPageResult{}, fmt.Errorf("match selection predicate: %w", err) + } + if matched { + v.Set(reflect.Append(v, reflect.ValueOf(obj).Elem())) + } + } + return listPageResult{pageLast: pageLast, count: count}, nil +} + +// ---- prepared writes ---- + +// preparedWrite is one fully-prepared object write: bytes and predicates only, +// so the gate holder has nothing to compute. +type preparedWrite struct { + key string + kind, namespace, name string + metadataJSON []byte + body []byte + rv int64 + uid string + // insert selects INSERT … ON CONFLICT DO NOTHING (create, or an + // ignoreNotFound update of an absent key) instead of the CAS UPDATE. + insert bool + expectRV int64 + expectUID string + // candidate is the object as persisted (RV bumped, checksum stamped); + // metaObj is its ObjectMeta-only copy for the lightweight watch event. + candidate runtime.Object + metaObj runtime.Object + // tsRow / baseKey are set for a TS-profile create: the time_series row + // that joins the transaction and the base key whose admission is + // re-checked inside it. + tsRow *TimeSeriesRow + baseKey string + // tsCompletion is the incoming TS profile's completion annotation, for + // the in-transaction admission rule. + tsCompletion string +} + +// stampAndEncode performs saveObject's pre-encode pipeline (RV bump, +// ManagedFields zeroing, checksum annotation) on obj and encodes it. +func (s *ObjectStore) stampAndEncode(key string, obj runtime.Object, checksum string) (*preparedWrite, error) { + version, err := s.versioner.ObjectResourceVersion(obj) + if err != nil { + return nil, fmt.Errorf("object resource version: %w", err) + } + if err := s.versioner.UpdateObject(obj, version+1); err != nil { + return nil, fmt.Errorf("set resourceVersion: %w", err) + } + managedFields := reflect.ValueOf(obj).Elem().FieldByName("ObjectMeta").FieldByName("ManagedFields") + if managedFields.IsValid() { + managedFields.Set(reflect.Zero(managedFields.Type())) + } + if checksum == "" { + checksum, err = s.calculateChecksum(obj) + if err != nil { + return nil, fmt.Errorf("calculate checksum: %w", err) + } + } + if anno := obj.(metav1.Object).GetAnnotations(); anno == nil { + obj.(metav1.Object).SetAnnotations(map[string]string{helpersv1.SyncChecksumMetadataKey: checksum}) + } else { + anno[helpersv1.SyncChecksumMetadataKey] = checksum + } + metaObj := extractFields(obj, []string{"ObjectMeta", "SchemaVersion"}) + metadataJSON, err := json.Marshal(metaObj) + if err != nil { + return nil, fmt.Errorf("marshal metadata: %w", err) + } + body, err := s.encodeBody(obj) + if err != nil { + return nil, err + } + _, _, kind, _, namespace, name := K8sPathToKeys(key) + return &preparedWrite{ + key: key, kind: kind, namespace: namespace, name: name, + metadataJSON: metadataJSON, + body: body, + rv: int64(version + 1), + uid: string(obj.(metav1.Object).GetUID()), + expectRV: int64(version), + expectUID: string(obj.(metav1.Object).GetUID()), + candidate: obj, + metaObj: metaObj, + }, nil +} + +// fillOut copies the persisted object into the REST layer's out parameter, +// exactly as saveObject does (shallow struct copy). +func fillOut(metaOut, candidate runtime.Object) { + if metaOut == nil { + return + } + val := reflect.ValueOf(metaOut) + if val.Kind() == reflect.Pointer { + val = val.Elem() + } + val.Set(reflect.ValueOf(candidate).Elem()) +} + +// applyTooLarge is the shared ObjectTooLargeError handling of Create and +// GuaranteedUpdate: clear the spec and mark the object, then save it anyway. +func applyTooLarge(obj runtime.Object) { + clearSpec(obj) + metadata := obj.(metav1.Object) + annotations := metadata.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + annotations[helpersv1.StatusMetadataKey] = helpersv1.TooLarge + metadata.SetAnnotations(annotations) +} + +// ---- the three transactions (§3.4), as statement lists ---- + +// statusPath / completionPath are json_extract paths into the metadata JSON. +// The internal ContainerProfile embeds ObjectMeta without a json tag, so +// encoding/json inlines its fields at the top level ("$.annotations", not +// "$.metadata.annotations"); the in-transaction admission check is therefore +// SQL and a string compare, not a Go decode (INV-1). The legacy store binds +// the JSON as a BLOB (json_extract would read it as JSONB and return NULL), +// hence the CAST; this store binds TEXT. +var ( + statusPath = fmt.Sprintf(`$.annotations."%s"`, helpersv1.StatusMetadataKey) + completionPath = fmt.Sprintf(`$.annotations."%s"`, helpersv1.CompletionMetadataKey) +) + +// execCreate runs Create's transaction body on conn. +func (s *ObjectStore) execCreate(conn *sqlite.Conn, pw *preparedWrite, hook func(string) error) error { + if pw.tsRow != nil && pw.baseKey != "" { + if err := hook("ts-admission"); err != nil { + return err + } + _, _, bkind, _, bns, bname := K8sPathToKeys(pw.baseKey) + var status, completion string + var found bool + err := sqlitex.Execute(conn, + `SELECT json_extract(CAST(metadata AS TEXT), :statusPath), json_extract(CAST(metadata AS TEXT), :completionPath) + FROM metadata WHERE kind = :kind AND namespace = :namespace AND name = :name`, + &sqlitex.ExecOptions{ + Named: map[string]any{":statusPath": statusPath, ":completionPath": completionPath, ":kind": bkind, ":namespace": bns, ":name": bname}, + ResultFunc: func(stmt *sqlite.Stmt) error { + found = true + status = stmt.ColumnText(0) + completion = stmt.ColumnText(1) + return nil + }, + }) + if err != nil { + return fmt.Errorf("ts admission read: %w", err) + } + if found { + switch { + case status == helpersv1.TooLarge: + return ObjectTooLargeError + case status == helpersv1.Completed && (completion == helpersv1.Full || pw.tsCompletion == helpersv1.Partial): + return ObjectCompletedError + } + } + } + if err := hook("insert-metadata"); err != nil { + return err + } + if err := sqlitex.Execute(conn, + `INSERT INTO metadata (kind, namespace, name, metadata, rv, uid, is_time_series) + VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING`, + &sqlitex.ExecOptions{Args: []any{pw.kind, pw.namespace, pw.name, string(pw.metadataJSON), pw.rv, pw.uid, pw.tsRow != nil}}); err != nil { + return fmt.Errorf("insert metadata: %w", err) + } + if conn.Changes() == 0 { + return storage.NewKeyExistsError(pw.key, 0) + } + if err := hook("insert-payload"); err != nil { + return err + } + if err := sqlitex.Execute(conn, + `INSERT INTO payloads (kind, namespace, name, encoding, body) VALUES (?, ?, ?, ?, ?)`, + &sqlitex.ExecOptions{Args: []any{pw.kind, pw.namespace, pw.name, PayloadEncodingJSONV1Beta1, pw.body}}); err != nil { + // A UNIQUE failure here is an orphan payloads row (K-2's shape). + return apierrors.NewInternalError(fmt.Errorf("insert payload for %q: %w", pw.key, err)) + } + if pw.tsRow != nil { + if err := hook("insert-time-series"); err != nil { + return err + } + if err := execTimeSeriesRow(conn, pw.tsRow); err != nil { + return err + } + } + return hook("commit") +} + +func execTimeSeriesRow(conn *sqlite.Conn, r *TimeSeriesRow) error { + return WriteTimeSeriesEntry(conn, r.Kind, r.Namespace, r.Name, r.SeriesID, r.TsSuffix, r.ReportTimestamp, r.Status, r.Completion, r.PreviousReportTimestamp, r.HasData) +} + +// execUpdate runs GuaranteedUpdate's transaction body on conn: the CAS UPDATE +// (or the create-or-conflict INSERT for an absent key), then the payloads +// UPDATE whose changes()==1 is asserted (K-6). +func (s *ObjectStore) execUpdate(conn *sqlite.Conn, pw *preparedWrite, hook func(string) error) error { + if pw.insert { + if err := hook("insert-metadata"); err != nil { + return err + } + if err := sqlitex.Execute(conn, + `INSERT INTO metadata (kind, namespace, name, metadata, rv, uid) + VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING`, + &sqlitex.ExecOptions{Args: []any{pw.kind, pw.namespace, pw.name, string(pw.metadataJSON), pw.rv, pw.uid}}); err != nil { + return fmt.Errorf("insert metadata: %w", err) + } + if conn.Changes() == 0 { + metrics.IncCPCASConflict("update") + return errWriteConflict + } + if err := hook("insert-payload"); err != nil { + return err + } + if err := sqlitex.Execute(conn, + `INSERT INTO payloads (kind, namespace, name, encoding, body) VALUES (?, ?, ?, ?, ?)`, + &sqlitex.ExecOptions{Args: []any{pw.kind, pw.namespace, pw.name, PayloadEncodingJSONV1Beta1, pw.body}}); err != nil { + return apierrors.NewInternalError(fmt.Errorf("insert payload for %q: %w", pw.key, err)) + } + return hook("commit") + } + if err := hook("update-metadata"); err != nil { + return err + } + if err := sqlitex.Execute(conn, + `UPDATE metadata SET metadata = ?, rv = ?, uid = ? + WHERE kind = ? AND namespace = ? AND name = ? AND rv = ? AND uid = ?`, + &sqlitex.ExecOptions{Args: []any{string(pw.metadataJSON), pw.rv, pw.uid, pw.kind, pw.namespace, pw.name, pw.expectRV, pw.expectUID}}); err != nil { + return fmt.Errorf("update metadata: %w", err) + } + if conn.Changes() == 0 { + metrics.IncCPCASConflict("update") + return errWriteConflict + } + if err := hook("update-payload"); err != nil { + return err + } + if err := sqlitex.Execute(conn, + `UPDATE payloads SET encoding = ?, body = ? WHERE kind = ? AND namespace = ? AND name = ?`, + &sqlitex.ExecOptions{Args: []any{PayloadEncodingJSONV1Beta1, pw.body, pw.kind, pw.namespace, pw.name}}); err != nil { + return fmt.Errorf("update payload: %w", err) + } + if n := conn.Changes(); n != 1 { + return apierrors.NewInternalError(fmt.Errorf("ObjectStore: metadata row for %q has %d payloads rows (INV-2 violated); transaction rolled back", pw.key, n)) + } + return hook("commit") +} + +// deleteResult carries the deleted metadata JSON out of the transaction so +// the decode into metaOut happens after COMMIT. +type deleteResult struct { + metadataJSON []byte + found bool +} + +// execDelete runs Delete's transaction body on conn. When expect is non-nil +// the metadata DELETE carries the per-object CAS (R4). +func (s *ObjectStore) execDelete(conn *sqlite.Conn, key string, expect *rowVersion, out *deleteResult, hook func(string) error) error { + _, _, kind, _, namespace, name := K8sPathToKeys(key) + if err := hook("delete-metadata"); err != nil { + return err + } + query := `DELETE FROM metadata WHERE kind = :kind AND namespace = :namespace AND name = :name RETURNING metadata` + named := map[string]any{":kind": kind, ":namespace": namespace, ":name": name} + if expect != nil { + query = `DELETE FROM metadata WHERE kind = :kind AND namespace = :namespace AND name = :name AND rv = :rv AND uid = :uid RETURNING metadata` + named[":rv"] = expect.rv + named[":uid"] = expect.uid + } + if err := sqlitex.Execute(conn, query, &sqlitex.ExecOptions{ + Named: named, + ResultFunc: func(stmt *sqlite.Stmt) error { + out.found = true + out.metadataJSON = []byte(stmt.ColumnText(0)) + return nil + }, + }); err != nil { + return fmt.Errorf("delete metadata: %w", err) + } + if !out.found { + if expect != nil { + // Distinguish "gone" from "changed under us": a row that still + // exists at another version is the R4 conflict. + var exists bool + if err := sqlitex.Execute(conn, + `SELECT 1 FROM metadata WHERE kind = :kind AND namespace = :namespace AND name = :name`, + &sqlitex.ExecOptions{ + Named: map[string]any{":kind": kind, ":namespace": namespace, ":name": name}, + ResultFunc: func(*sqlite.Stmt) error { exists = true; return nil }, + }); err != nil { + return fmt.Errorf("delete metadata re-check: %w", err) + } + if exists { + metrics.IncCPCASConflict("delete") + return errWriteConflict + } + } + return storage.NewKeyNotFoundError(key, 0) + } + if err := hook("delete-payload"); err != nil { + return err + } + if err := sqlitex.Execute(conn, + `DELETE FROM payloads WHERE kind = ? AND namespace = ? AND name = ?`, + &sqlitex.ExecOptions{Args: []any{kind, namespace, name}}); err != nil { + return fmt.Errorf("delete payload: %w", err) + } + if IsContainerProfileKind(kind) { + if err := hook("delete-time-series"); err != nil { + return err + } + if err := DeleteTimeSeriesContainerEntries(conn, key); err != nil { + return err + } + } + return hook("commit") +} + +// ---- Create ---- + +// Create implements storage.Interface (§3.4 Create). +func (s *ObjectStore) Create(ctx context.Context, key string, obj, metaOut runtime.Object, _ uint64) error { + if version, err := s.versioner.ObjectResourceVersion(obj); err == nil && version != 0 { + msg := "resourceVersion should not be set on objects to be created" + logger.L().Ctx(ctx).Error(msg) + return errors.New(msg) + } + defer s.reservations.enter(ctx, key)() + + // prepare (caller goroutine, no transaction, one pool connection for reads) + conn, err := s.takeConn(ctx, "create", key) + if err != nil { + return err + } + h := &readHandle{store: s, conn: conn} + presaveCtx := withReadHandle(ctx, h) + if err := s.processor.PreSave(presaveCtx, obj); err != nil { + if errors.Is(err, ObjectTooLargeError) { + applyTooLarge(obj) + logger.L().Debug("Create - too large object, saving metadata only", helpers.String("key", key)) + } else { + s.pool.Put(conn) + return err + } + } + s.pool.Put(conn) + + pw, err := s.stampAndEncode(key, obj, "") + if err != nil { + return err + } + provider, hasProvider := s.processor.(TimeSeriesRowProvider) + if hasProvider { + if row, baseKey, ok := provider.TimeSeriesRowFor(obj); ok { + pw.tsRow = &row + pw.baseKey = baseKey + pw.tsCompletion = obj.(metav1.Object).GetAnnotations()[helpersv1.CompletionMetadataKey] + } + } + if s.hooks.afterPrepare != nil { + s.hooks.afterPrepare(holdPathCreate, key) + } + + // transaction (gate held) + err = s.gate.run(ctx, priorityHigh, holdPathCreate, ContainerProfileKindPlural, func(_ context.Context, conn *sqlite.Conn) error { + return s.execCreate(conn, pw, s.stmtHook(conn, holdPathCreate)) + }) + if err != nil { + if !storage.IsExist(err) && !errors.Is(err, ObjectCompletedError) && !errors.Is(err, ObjectTooLargeError) { + logger.L().Ctx(ctx).Error("Create - save object failed", helpers.Error(err), helpers.String("key", key)) + } + return err + } + s.checkpointer.afterCommit() + + // after (caller goroutine, gate released) + if !hasProvider { + conn2, err := s.takeConn(ctx, "create", key) + if err != nil { + return err + } + afterCtx := withReadHandle(ctx, &readHandle{store: s, conn: conn2}) + err = s.processor.AfterCreate(afterCtx, pw.candidate) + s.pool.Put(conn2) + if err != nil { + return fmt.Errorf("processor.AfterCreate: %w", err) + } + } + fillOut(metaOut, pw.candidate) + s.watchDispatcher.Added(key, pw.metaObj, pw.candidate) + return nil +} + +// stmtHook returns the per-transaction statement-boundary hook for path: a +// counter over the statements of ONE gated transaction, feeding the +// beforeStatement seam. Nil-cheap in production. +func (s *ObjectStore) stmtHook(conn *sqlite.Conn, path string) func(name string) error { + if s.hooks.beforeStatement == nil { + return noHook + } + idx := 0 + return func(name string) error { + idx++ + return s.hooks.beforeStatement(conn, path, idx, name) + } +} + +func noHook(string) error { return nil } + +// ---- GuaranteedUpdate ---- + +// updateState is one read of the current object with the row version the +// payload came from: the CAS expectation. +type updateState struct { + obj runtime.Object + rev int64 + uid string + exists bool +} + +// readState reads key on conn into a fresh object of v's type. +func (s *ObjectStore) readState(ctx context.Context, conn *sqlite.Conn, key string, ignoreNotFound bool, v reflect.Value) (*updateState, error) { + objPtr := reflect.New(v.Type()).Interface().(runtime.Object) + row, err := s.getWithConn(ctx, conn, key, storage.GetOptions{IgnoreNotFound: ignoreNotFound}, objPtr) + if err != nil { + return nil, err + } + st := &updateState{obj: objPtr} + if row != nil { + st.exists = true + st.rev = row.rv + st.uid = row.uid + } + return st, nil +} + +// prepareUpdate is guaranteedUpdateSingleWriter's prepare phase: preconditions, +// tryUpdate, PreSave on both sides, the #315 DeepEqual short-circuit, then +// stamp + encode with the CAS expectation taken from the row the payload came +// from. It returns (nil, state, nil) for a no-op update; (pw, state, nil) for a +// write to commit; the loop over conflicts belongs to the caller. +// +// Returned errors: retryable staleness is signalled by errStaleState. +var errStaleState = errors.New("stale state") + +func (s *ObjectStore) prepareUpdate(ctx context.Context, h *readHandle, key string, ignoreNotFound bool, + preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, origState *updateState, origStateIsCurrent bool, checksum string) (*preparedWrite, error) { + + if err := preconditions.Check(key, origState.obj); err != nil { + if origStateIsCurrent { + logger.L().Ctx(ctx).Error("GuaranteedUpdate - preconditions check failed", helpers.Error(err), helpers.String("key", key)) + return nil, err + } + return nil, errStaleState + } + + orig := origState.obj.DeepCopyObject() + presaveCtx := withReadHandle(ctx, h) + _ = s.processor.PreSave(presaveCtx, orig) + + ret, _, err := tryUpdate(origState.obj, storage.ResponseMeta{}) + if err != nil { + if origStateIsCurrent { + if !apierrors.IsNotFound(err) && !apierrors.IsInvalid(err) { + logger.L().Ctx(ctx).Error("GuaranteedUpdate - tryUpdate func failed", helpers.Error(err), helpers.String("key", key)) + } + return nil, err + } + return nil, fmt.Errorf("%w: %w", errStaleState, err) + } + + if err := s.processor.PreSave(presaveCtx, ret); err != nil { + if errors.Is(err, ObjectTooLargeError) { + applyTooLarge(ret) + logger.L().Debug("GuaranteedUpdate - too large object, skipping update", helpers.String("key", key)) + } else { + logger.L().Debug("GuaranteedUpdate - processor.PreSave failed", helpers.Error(err), helpers.String("key", key)) + return nil, err + } + } + + if reflect.DeepEqual(orig, ret) { + logger.L().Debug("GuaranteedUpdate - tryUpdate returned the same object, no update needed", helpers.String("key", key)) + return nil, nil + } + + pw, err := s.stampAndEncode(key, ret, checksum) + if err != nil { + return nil, err + } + pw.expectRV = origState.rev + pw.expectUID = origState.uid + pw.insert = !origState.exists + return pw, nil +} + +// GuaranteedUpdate implements storage.Interface (§3.4 Update): read → prepare → +// one CAS transaction under the gate; on conflict, backoff, re-read, retry. +func (s *ObjectStore) GuaranteedUpdate(ctx context.Context, key string, metaOut runtime.Object, ignoreNotFound bool, + preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, cachedExistingObject runtime.Object) error { + return s.guaranteedUpdate(ctx, key, metaOut, ignoreNotFound, preconditions, tryUpdate, cachedExistingObject, "", priorityHigh) +} + +func (s *ObjectStore) guaranteedUpdate(ctx context.Context, key string, metaOut runtime.Object, ignoreNotFound bool, + preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, cachedExistingObject runtime.Object, + checksum string, priority writePriority) error { + + v, err := conversion.EnforcePtr(metaOut) + if err != nil { + return fmt.Errorf("unable to convert output object to pointer: %v", err) + } + defer s.reservations.enter(ctx, key)() + + conn, err := s.takeConn(ctx, "update", key) + if err != nil { + return err + } + defer s.pool.Put(conn) + h := &readHandle{store: s, conn: conn} + + var origState *updateState + var origStateIsCurrent bool + if cachedExistingObject != nil { + rv, err := s.versioner.ObjectResourceVersion(cachedExistingObject) + if err != nil { + return fmt.Errorf("couldn't get resource version: %v", err) + } + origState = &updateState{obj: cachedExistingObject, rev: int64(rv), uid: string(cachedExistingObject.(metav1.Object).GetUID()), exists: rv != 0} + } else { + origState, err = s.readState(ctx, conn, key, ignoreNotFound, v) + if err != nil { + return err + } + origStateIsCurrent = true + } + + if annotations := origState.obj.(metav1.Object).GetAnnotations(); annotations != nil && annotations[helpersv1.StatusMetadataKey] == helpersv1.TooLarge { + logger.L().Debug("GuaranteedUpdate - already too large object, skipping update", helpers.String("key", key)) + v.Set(reflect.ValueOf(origState.obj).Elem()) + return nil + } + + var conflictAttempts int + for { + pw, err := s.prepareUpdate(ctx, h, key, ignoreNotFound, preconditions, tryUpdate, origState, origStateIsCurrent, checksum) + if errors.Is(err, errStaleState) { + origState, err = s.readState(ctx, conn, key, ignoreNotFound, v) + if err != nil { + return err + } + origStateIsCurrent = true + continue + } + if err != nil { + return err + } + if pw == nil { + v.Set(reflect.ValueOf(origState.obj).Elem()) + return nil + } + if s.hooks.afterPrepare != nil { + s.hooks.afterPrepare(holdPathUpdate, key) + } + + err = s.gate.run(ctx, priority, holdPathUpdate, ContainerProfileKindPlural, func(_ context.Context, conn *sqlite.Conn) error { + return s.execUpdate(conn, pw, s.stmtHook(conn, holdPathUpdate)) + }) + if errors.Is(err, errWriteConflict) { + conflictAttempts++ + metrics.IncSingleWriterConflictRetry(resourceFromKey(key)) + if backoffErr := singleWriterConflictBackoff(ctx, conflictAttempts); backoffErr != nil { + return newContentionTimeoutError("update", key, backoffErr) + } + origState, err = s.readState(ctx, conn, key, ignoreNotFound, v) + if err != nil { + return err + } + origStateIsCurrent = true + continue + } + if err != nil { + logger.L().Ctx(ctx).Error("GuaranteedUpdate - save object failed", helpers.Error(err), helpers.String("key", key)) + return err + } + s.checkpointer.afterCommit() + fillOut(metaOut, pw.candidate) + s.watchDispatcher.Modified(key, pw.metaObj, pw.candidate) + return nil + } +} + +// ---- Delete ---- + +// Delete implements storage.Interface (§3.4 Delete): metadata + payloads + +// time_series rows removed in one transaction; KeyNotFound when no row. +func (s *ObjectStore) Delete(ctx context.Context, key string, metaOut runtime.Object, _ *storage.Preconditions, _ storage.ValidateObjectFunc, _ runtime.Object, _ storage.DeleteOptions) error { + return s.deleteKey(ctx, key, metaOut, nil, priorityHigh) +} + +func (s *ObjectStore) deleteKey(ctx context.Context, key string, metaOut runtime.Object, expect *rowVersion, priority writePriority) error { + defer s.reservations.enter(ctx, key)() + var res deleteResult + err := s.gate.run(ctx, priority, holdPathDelete, ContainerProfileKindPlural, func(_ context.Context, conn *sqlite.Conn) error { + return s.execDelete(conn, key, expect, &res, s.stmtHook(conn, holdPathDelete)) + }) + if err != nil { + if !storage.IsNotFound(err) { + logger.L().Ctx(ctx).Error("Delete - delete failed", helpers.Error(err), helpers.String("key", key)) + } + return err + } + s.checkpointer.afterCommit() + if metaOut != nil { + if err := json.Unmarshal(res.metadataJSON, metaOut); err != nil { + return fmt.Errorf("unmarshal deleted metadata: %w", err) + } + } + s.watchDispatcher.Deleted(key, metaOut) + return nil +} diff --git a/pkg/registry/file/sqliteobject_store_test.go b/pkg/registry/file/sqliteobject_store_test.go new file mode 100644 index 000000000..ef8f56f59 --- /dev/null +++ b/pkg/registry/file/sqliteobject_store_test.go @@ -0,0 +1,493 @@ +package file + +import ( + "bytes" + "context" + "encoding/gob" + "encoding/json" + "errors" + "fmt" + "testing" + "time" + + helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apiserver/pkg/storage" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitex" +) + +func identityTryUpdate(input runtime.Object, _ storage.ResponseMeta) (runtime.Object, *uint64, error) { + return input, nil, nil +} + +func setLabel(k, v string) storage.UpdateFunc { + return func(input runtime.Object, _ storage.ResponseMeta) (runtime.Object, *uint64, error) { + cp := input.(*softwarecomposition.ContainerProfile) + if cp.Labels == nil { + cp.Labels = map[string]string{} + } + cp.Labels[k] = v + return cp, nil, nil + } +} + +func TestObjectStore_CreateGetUpdateDeleteRoundTrip(t *testing.T) { + e := newObjectStoreEnv(t) + p := e.plain("rt") + p.TypeMeta = metav1.TypeMeta{APIVersion: StorageV1Beta1ApiVersion, Kind: "ContainerProfile"} + key := e.key("rt") + + created := e.create(p) + assert.Equal(t, "1", created.ResourceVersion) + assert.NotEmpty(t, created.Annotations[helpersv1.SyncChecksumMetadataKey]) + e.withConn(func(c *sqlite.Conn) { assertINV2(t, c, key) }) + + got := e.mustGet(key) + assert.Equal(t, created.ResourceVersion, got.ResourceVersion) + assert.Equal(t, created.UID, got.UID) + assert.Equal(t, p.TypeMeta, got.TypeMeta, "TypeMeta must survive the JSON body") + assert.Equal(t, canonicalCP(created).Spec, canonicalCP(got).Spec) + assert.Equal(t, created.Annotations, got.Annotations) + + out := &softwarecomposition.ContainerProfile{} + require.NoError(t, e.store.GuaranteedUpdate(e.ctx, key, out, false, nil, setLabel("a", "b"), nil)) + assert.Equal(t, "2", out.ResourceVersion) + assert.Equal(t, "b", e.mustGet(key).Labels["a"]) + e.withConn(func(c *sqlite.Conn) { assertINV2(t, c, key) }) + + // no-op update: no write, no RV bump (#315) + require.NoError(t, e.store.GuaranteedUpdate(e.ctx, key, out, false, nil, identityTryUpdate, nil)) + assert.Equal(t, "2", out.ResourceVersion) + assert.Equal(t, "2", e.mustGet(key).ResourceVersion) + + deleted := &softwarecomposition.ContainerProfile{} + require.NoError(t, e.store.Delete(e.ctx, key, deleted, nil, nil, nil, storage.DeleteOptions{})) + assert.Equal(t, "2", deleted.ResourceVersion) + _, err := e.get(key) + assert.True(t, storage.IsNotFound(err), "%v", err) + row := e.inspect(key) + assert.False(t, row.metaExists) + assert.False(t, row.payloadExists) + + err = e.store.Delete(e.ctx, key, &softwarecomposition.ContainerProfile{}, nil, nil, nil, storage.DeleteOptions{}) + assert.True(t, storage.IsNotFound(err), "delete of an absent key: %v", err) +} + +// TestObjectStore_CreateStates covers Create over (absent, present, +// present-being-recreated): the second create of a key is KeyExists; a +// delete+create restarts the key at rv=1 with a fresh UID. +func TestObjectStore_CreateStates(t *testing.T) { + e := newObjectStoreEnv(t) + key := e.key("cs") + first := e.create(e.plain("cs")) + assert.Equal(t, "1", first.ResourceVersion) + + err := e.store.Create(e.ctx, key, e.plain("cs"), nil, 0) + assert.True(t, storage.IsExist(err), "%v", err) + assert.Equal(t, first.UID, e.mustGet(key).UID, "a refused create must not touch the row") + + require.NoError(t, e.store.Delete(e.ctx, key, nil, nil, nil, nil, storage.DeleteOptions{})) + second := e.create(e.plain("cs")) + assert.Equal(t, "1", second.ResourceVersion) + assert.NotEqual(t, first.UID, second.UID) + + err = e.store.Create(e.ctx, key, &softwarecomposition.ContainerProfile{ObjectMeta: metav1.ObjectMeta{Name: "cs", ResourceVersion: "7"}}, nil, 0) + require.Error(t, err) + assert.Contains(t, err.Error(), "resourceVersion should not be set") +} + +// TestObjectStore_CASMatrix drives the compare-and-swap statement directly over +// (row state) × (expectation): the row is absent, present at (rv 1, uid A), or +// recreated at (rv 1, uid B) after a delete; the expectation matches, is +// stale, or names a different uid. Exactly the matching cases commit. +func TestObjectStore_CASMatrix(t *testing.T) { + e := newObjectStoreEnv(t) + uidA, uidB := "uid-a", "uid-b" + body, err := e.store.encodeBody(e.plain("cas")) + require.NoError(t, err) + + type expectation struct { + name string + insert bool + expectRV int64 + expectUID string + } + type rowState struct { + name string + seed func(conn *sqlite.Conn) + } + seedRow := func(uid string) func(conn *sqlite.Conn) { + return func(conn *sqlite.Conn) { + meta := fmt.Sprintf(`{"name":"cas","namespace":%q,"uid":%q,"resourceVersion":"1"}`, e.ns, uid) + require.NoError(t, sqlitex.Execute(conn, `INSERT INTO metadata (kind,namespace,name,metadata,rv,uid) VALUES ('containerprofile',?, 'cas', ?, 1, ?)`, + &sqlitex.ExecOptions{Args: []any{e.ns, meta, uid}})) + require.NoError(t, sqlitex.Execute(conn, `INSERT INTO payloads (kind,namespace,name,encoding,body) VALUES ('containerprofile',?, 'cas', ?, ?)`, + &sqlitex.ExecOptions{Args: []any{e.ns, PayloadEncodingJSONV1Beta1, body}})) + } + } + states := []rowState{ + {"absent", func(*sqlite.Conn) {}}, + {"present-rv1-uidA", seedRow(uidA)}, + {"recreated-rv1-uidB", func(conn *sqlite.Conn) { seedRow(uidA)(conn); wipe(t, conn, e.ns, "cas"); seedRow(uidB)(conn) }}, + } + expectations := []expectation{ + {"insert-if-absent", true, 0, ""}, + {"rv1-uidA", false, 1, uidA}, + {"rv0-uidA (stale)", false, 0, uidA}, + {"rv1-uidB", false, 1, uidB}, + } + // which (state, expectation) pairs commit + commits := map[string]bool{ + "absent/insert-if-absent": true, + "present-rv1-uidA/rv1-uidA": true, + "recreated-rv1-uidB/rv1-uidB": true, + } + for _, st := range states { + for _, ex := range expectations { + name := st.name + "/" + ex.name + t.Run(name, func(t *testing.T) { + e.withFixture(func(conn *sqlite.Conn) { + wipe(t, conn, e.ns, "cas") + st.seed(conn) + pw := &preparedWrite{key: e.key("cas"), kind: "containerprofile", namespace: e.ns, name: "cas", + metadataJSON: []byte(fmt.Sprintf(`{"name":"cas","namespace":%q,"uid":%q,"resourceVersion":"%d"}`, e.ns, "uid-new", ex.expectRV+1)), + body: body, rv: ex.expectRV + 1, uid: "uid-new", insert: ex.insert, expectRV: ex.expectRV, expectUID: ex.expectUID} + if ex.insert { + pw.rv, pw.uid = 1, "uid-new" + pw.metadataJSON = []byte(fmt.Sprintf(`{"name":"cas","namespace":%q,"uid":"uid-new","resourceVersion":"1"}`, e.ns)) + } + endFn, err := sqlitex.ImmediateTransaction(conn) + require.NoError(t, err) + err = e.store.execUpdate(conn, pw, noHook) + endFn(&err) + if commits[name] { + require.NoError(t, err) + row := inspectRow(t, conn, e.key("cas")) + require.NotNil(t, row.rv) + assert.Equal(t, pw.rv, *row.rv) + assert.Equal(t, "uid-new", *row.uid) + } else { + assert.True(t, errors.Is(err, errWriteConflict), "expected conflict, got %v", err) + } + assertINV2(t, conn, e.key("cas")) + }) + }) + } + } +} + +func wipe(t *testing.T, conn *sqlite.Conn, ns, name string) { + t.Helper() + require.NoError(t, sqlitex.Execute(conn, `DELETE FROM metadata WHERE kind='containerprofile' AND namespace=? AND name=?`, &sqlitex.ExecOptions{Args: []any{ns, name}})) + require.NoError(t, sqlitex.Execute(conn, `DELETE FROM payloads WHERE kind='containerprofile' AND namespace=? AND name=?`, &sqlitex.ExecOptions{Args: []any{ns, name}})) +} + +// TestObjectStore_TSAdmissionInsideTransaction: the base's four states. The +// authoritative check is the in-transaction SELECT, so the base is flipped +// AFTER the prepare phase's PreSave passed (design "gap 1") and the Create +// must still fail — with no TS object and no time_series row left behind. +func TestObjectStore_TSAdmissionInsideTransaction(t *testing.T) { + cases := []struct { + name string + baseStatus string + baseComplete string + incoming string + wantErr error + wantAdmitted bool + }{ + {"absent", "", "", helpersv1.Partial, nil, true}, + {"learning", helpersv1.Learning, helpersv1.Partial, helpersv1.Partial, nil, true}, + {"completed-partial+full-incoming", helpersv1.Completed, helpersv1.Partial, helpersv1.Full, nil, true}, + {"completed-partial+partial-incoming", helpersv1.Completed, helpersv1.Partial, helpersv1.Partial, ObjectCompletedError, false}, + {"completed-full", helpersv1.Completed, helpersv1.Full, helpersv1.Full, ObjectCompletedError, false}, + {"too-large", helpersv1.TooLarge, helpersv1.Partial, helpersv1.Partial, ObjectTooLargeError, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + e := newObjectStoreEnv(t) + if tc.baseStatus != "" { + // Flip the base between prepare and commit: PreSave sees no base + // (or a Learning one) and admits; the transaction must not. + e.store.hooks.afterPrepare = func(path, key string) { + if path != holdPathCreate { + return + } + e.store.hooks.afterPrepare = nil + base := e.plain(e.baseNm) + base.Annotations[helpersv1.StatusMetadataKey] = tc.baseStatus + base.Annotations[helpersv1.CompletionMetadataKey] = tc.baseComplete + e.create(base) + } + } + ts := e.ts("r1", 1, helpersv1.Learning, tc.incoming) + err := e.store.Create(e.ctx, e.tsKey("r1"), ts, nil, 0) + row := e.inspect(e.tsKey("r1")) + base := e.inspect(e.baseKey) + if tc.wantAdmitted { + require.NoError(t, err) + assert.True(t, row.metaExists && row.payloadExists) + assert.Equal(t, 1, base.tsRows, "the time_series row joins the create") + } else { + require.ErrorIs(t, err, tc.wantErr) + assert.False(t, row.metaExists, "refused create must leave no metadata row") + assert.False(t, row.payloadExists, "refused create must leave no payload") + assert.Equal(t, 0, base.tsRows, "refused create must leave no time_series row") + } + }) + } +} + +// TestObjectStore_K6_PayloadRowMissingFailsLoudly: a metadata row without its +// payloads row (INV-2 already violated by something else) makes the payloads +// UPDATE change 0 rows; the transaction rolls back and the caller gets an +// InternalError, instead of a metadata row silently advancing alone. +func TestObjectStore_K6_PayloadRowMissingFailsLoudly(t *testing.T) { + e := newObjectStoreEnv(t) + key := e.key("k6") + e.create(e.plain("k6")) + e.withFixture(func(conn *sqlite.Conn) { + require.NoError(t, sqlitex.Execute(conn, `DELETE FROM payloads WHERE name='k6'`, nil)) + }) + // The read needs a payload; feed the update the cached object so the + // prepare phase does not fail first. + cached := e.plain("k6") + cached.ResourceVersion = "1" + cached.UID = types.UID(e.inspect(key).uidOrEmpty()) + err := e.store.GuaranteedUpdate(e.ctx, key, &softwarecomposition.ContainerProfile{}, false, nil, setLabel("x", "y"), cached) + require.Error(t, err) + assert.True(t, apierrors.IsInternalError(err), "%v", err) + assert.Contains(t, err.Error(), "INV-2") + row := e.inspect(key) + require.NotNil(t, row.rv) + assert.Equal(t, int64(1), *row.rv, "metadata UPDATE must have been rolled back") + assert.True(t, e.store.gate.conn.AutocommitEnabled(), "gate connection left in a transaction") +} + +func (r dbRow) uidOrEmpty() string { + if r.uid == nil { + return "" + } + return *r.uid +} + +// TestObjectStore_K3_AutocheckpointOffOnEveryConnection: wal_autocheckpoint is +// a per-connection setting; PoolOptions.DisableAutoCheckpoint must reach every +// connection of the pool, not only the gate's. +func TestObjectStore_K3_AutocheckpointOffOnEveryConnection(t *testing.T) { + check := func(t *testing.T, disable bool, want int64) { + pool := NewPoolWithOptions(t.TempDir()+"/k3.sq3", PoolOptions{Size: 4, DisableAutoCheckpoint: disable}) + var conns []*sqlite.Conn + defer func() { + for _, c := range conns { + pool.Put(c) + } + _ = pool.Close() + }() + for i := 0; i < 4; i++ { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + c, err := pool.Take(ctx) + require.NoError(t, err) + c.SetInterrupt(nil) + cancel() + conns = append(conns, c) + var v int64 = -1 + require.NoError(t, sqlitex.ExecuteTransient(c, `PRAGMA wal_autocheckpoint`, &sqlitex.ExecOptions{ResultFunc: func(stmt *sqlite.Stmt) error { v = stmt.ColumnInt64(0); return nil }})) + assert.Equal(t, want, v, "connection %d", i) + } + } + t.Run("disabled", func(t *testing.T) { check(t, true, 0) }) + t.Run("default", func(t *testing.T) { check(t, false, 1000) }) +} + +// TestObjectStore_K5_CloseReturnsGateConnection: the gate's connection is back +// in the pool after Close, so every one of the pool's connections can be taken +// and Pool.Close does not block. (The env's cleanup asserts Close returns.) +func TestObjectStore_K5_CloseReturnsGateConnection(t *testing.T) { + e := newObjectStoreEnv(t, withPoolSize(3)) + countTakeable := func() int { + var conns []*sqlite.Conn + defer func() { + for _, c := range conns { + e.pool.Put(c) + } + }() + for len(conns) < 10 { + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + c, err := e.pool.Take(ctx) + cancel() + if err != nil { + break + } + conns = append(conns, c) + } + return len(conns) + } + assert.Equal(t, 2, countTakeable(), "one of three connections is the gate's while the gate is open") + // The shared gate is closed by its owner after every store's Close (the + // apiserver's pre-shutdown hook); the store's own Close returns nothing. + require.NoError(t, e.store.Close()) + assert.Equal(t, 2, countTakeable(), "the store does not own the gate's connection") + require.NoError(t, e.gate.Close()) + assert.Equal(t, 3, countTakeable(), "the gate's Close must return its connection") + require.NoError(t, e.store.Close(), "Close is idempotent") + require.NoError(t, e.gate.Close(), "Close is idempotent") + err := e.store.Create(e.ctx, e.key("after-close"), e.plain("after-close"), nil, 0) + assert.ErrorIs(t, err, errGateClosed) +} + +// TestObjectStore_CodecFidelity round-trips a user-authored multi-container +// profile (the subtype-groups regression) and pins the ONE fidelity delta the +// design lists: creationTimestamp is truncated to whole seconds. +func TestObjectStore_CodecFidelity(t *testing.T) { + e := newObjectStoreEnv(t) + p := groupedUserCP(e.ns, "grouped") + p.UID = uuid.NewUUID() + stamp := metav1.NewTime(time.Date(2026, 9, 8, 12, 0, 0, 123456789, time.UTC)) + p.CreationTimestamp = stamp + key := e.key("grouped") + created := e.create(p) + got := e.mustGet(key) + + assert.Equal(t, created.Spec.Containers, got.Spec.Containers) + assert.Equal(t, created.Spec.InitContainers, got.Spec.InitContainers) + assert.Equal(t, created.Spec.EphemeralContainers, got.Spec.EphemeralContainers) + assert.True(t, got.CreationTimestamp.Time.Equal(stamp.Truncate(time.Second)), + "creationTimestamp truncated to seconds (intended divergence): got %s", got.CreationTimestamp.Time) + assert.False(t, got.CreationTimestamp.Time.Equal(stamp.Time), "the sub-second part is gone") + // What the legacy store's GET returns is the gob round trip of the same + // persisted object (empty collections come back nil on both codecs). + var buf bytes.Buffer + require.NoError(t, gob.NewEncoder(&buf).Encode(created)) + legacyView := &softwarecomposition.ContainerProfile{} + require.NoError(t, gob.NewDecoder(&buf).Decode(legacyView)) + assert.True(t, legacyView.CreationTimestamp.Time.Equal(stamp.Time), "gob keeps the nanoseconds") + // Second codec delta, NOT in the design's list (reported): v1beta1 spec + // collections without omitempty come back as empty slices from JSON and + // as nil from gob (REST renders them as [] vs null). + assert.Nil(t, legacyView.Spec.Execs, "gob: empty Execs decodes to nil") + assert.NotNil(t, got.Spec.Execs, "json: empty Execs decodes to an empty slice") + assert.Empty(t, got.Spec.Execs) + assert.Equal(t, canonicalCP(legacyView), canonicalCP(got), "no field beyond creationTimestamp precision and nil-vs-empty collections may differ from the legacy GET") + + // v1beta1 round trip of the body itself + var v1 map[string]any + e.withConn(func(conn *sqlite.Conn) { + require.NoError(t, sqlitex.Execute(conn, `SELECT body, encoding FROM payloads WHERE name='grouped'`, &sqlitex.ExecOptions{ResultFunc: func(stmt *sqlite.Stmt) error { + assert.Equal(t, PayloadEncodingJSONV1Beta1, stmt.ColumnText(1)) + return json.Unmarshal([]byte(stmt.ColumnText(0)), &v1) + }})) + }) + assert.NotNil(t, v1["spec"], "body is the v1beta1 wire form (lower-case json tags)") +} + +// TestObjectStore_ListPagination: metadata LIST is the legacy statement; the +// fullSpec LIST is a paginated join; an UPDATE mid-list keeps the rowid so a +// paginating client sees each object exactly once. +func TestObjectStore_ListPagination(t *testing.T) { + e := newObjectStoreEnv(t) + for _, n := range []string{"a", "b", "c"} { + e.create(e.plain("pg-" + n)) + } + listKey := testCPPrefix + e.ns + page := func(rv, cont string, limit int64) *softwarecomposition.ContainerProfileList { + out := &softwarecomposition.ContainerProfileList{} + opts := storage.ListOptions{ResourceVersion: rv, Predicate: storage.SelectionPredicate{Limit: limit, Continue: cont}, Recursive: true} + require.NoError(t, e.store.GetList(e.ctx, listKey, opts, out)) + return out + } + names := func(l *softwarecomposition.ContainerProfileList) []string { + var out []string + for _, it := range l.Items { + out = append(out, it.Name) + } + return out + } + for _, rv := range []string{softwarecomposition.ResourceVersionMetadata, softwarecomposition.ResourceVersionFullSpec} { + p1 := page(rv, "", 2) + assert.Equal(t, []string{"pg-a", "pg-b"}, names(p1), rv) + require.NotEmpty(t, p1.Continue) + require.NoError(t, e.store.GuaranteedUpdate(e.ctx, e.key("pg-a"), &softwarecomposition.ContainerProfile{}, false, nil, setLabel("k", rv), nil)) + p2 := page(rv, p1.Continue, 2) + assert.Equal(t, []string{"pg-c"}, names(p2), "%s: updated object must not reappear (rowid stable)", rv) + assert.Empty(t, p2.Continue) + } + full := page(softwarecomposition.ResourceVersionFullSpec, "", 0) + require.Len(t, full.Items, 3) + assert.NotEmpty(t, full.Items[0].Spec.Architectures, "fullSpec list carries the body") + meta := page(softwarecomposition.ResourceVersionMetadata, "", 0) + assert.Empty(t, meta.Items[0].Spec.Architectures, "metadata list carries no body") +} + +// TestObjectStore_ListExcludesUnconsolidatedTSRows: an un-consolidated TS +// profile create writes a real metadata+payloads row (see the ObjectStore +// file comment), one per report, alongside its time_series bookkeeping row. +// Those rows are internal to consolidation and must never surface through +// the k8s List API -- a client that never asked about time-series internals +// should see exactly the consolidated/base objects. This also is the fix for +// Tier B's list-p95-ms regression (§ keyReserveWaitMax tuning history): more +// writers proceeding unreserved past an in-flight consolidation retry left +// more un-merged TS rows sitting in `metadata` before the tick could clear +// them, and List had no way to skip them -- it decoded every one. +func TestObjectStore_ListExcludesUnconsolidatedTSRows(t *testing.T) { + e := newObjectStoreEnv(t) + base := e.create(e.plain("plain-object")) + for i := 1; i <= 3; i++ { + e.create(e.ts(fmt.Sprintf("s%d", i), i, helpersv1.Learning, helpersv1.Partial)) + } + listKey := testCPPrefix + e.ns + for _, rv := range []string{softwarecomposition.ResourceVersionMetadata, softwarecomposition.ResourceVersionFullSpec} { + out := &softwarecomposition.ContainerProfileList{} + opts := storage.ListOptions{ResourceVersion: rv, Predicate: storage.SelectionPredicate{Limit: 0}, Recursive: true} + require.NoError(t, e.store.GetList(e.ctx, listKey, opts, out)) + var names []string + for _, it := range out.Items { + names = append(names, it.Name) + } + assert.Equal(t, []string{base.Name}, names, "%s: only the consolidated/base object is listed, not the pending TS rows", rv) + } + // The TS rows are real rows, not silently dropped -- ConsolidateTimeSeries + // still finds and merges them normally into their own series (e.baseKey, + // unrelated to the plain object above); List's exclusion is read-side only. + e.tick() + assert.Equal(t, 0, e.pendingTSRows(e.baseKey), "consolidation still sees and merges the excluded TS rows") + assert.Equal(t, helpersv1.Learning, e.mustGet(e.baseKey).Annotations[helpersv1.StatusMetadataKey]) +} + +// TestObjectStore_ListFullSpec_MissingPayloadRowDoesNotHideLaterRows: a +// metadata row with no payloads row (e.g. left by a migration path for an +// undecodable legacy file) sorts before two valid rows. fetchListPage's +// fullSpec query paginates metadata by rowid, LIMIT :limit, THEN inner-joins +// payloads (PM-1: apply the predicate before the join) -- a page whose +// metadata scan includes the missing-payload row used to return fewer +// objects than :limit purely because the join dropped one, and GetList's +// "count < remaining -> EOF" check took that as real end-of-data, clearing +// the continuation token and silently hiding every row after the gap +// forever (not just on this call: a fresh List from cursor "" hits the same +// row first every time). +func TestObjectStore_ListFullSpec_MissingPayloadRowDoesNotHideLaterRows(t *testing.T) { + e := newObjectStoreEnv(t) + e.withFixture(func(conn *sqlite.Conn) { + require.NoError(t, WriteJSON(conn, e.key("no-payload"), []byte(`{"name":"no-payload","namespace":"`+e.ns+`"}`))) + }) + e.create(e.plain("valid-a")) + e.create(e.plain("valid-b")) + + listKey := testCPPrefix + e.ns + out := &softwarecomposition.ContainerProfileList{} + opts := storage.ListOptions{ResourceVersion: softwarecomposition.ResourceVersionFullSpec, Predicate: storage.SelectionPredicate{Limit: 2}, Recursive: true} + require.NoError(t, e.store.GetList(e.ctx, listKey, opts, out)) + var names []string + for _, it := range out.Items { + names = append(names, it.Name) + } + require.ElementsMatch(t, []string{"valid-a", "valid-b"}, names, + "both valid rows must be reachable; before the fix, valid-b was permanently hidden by the gap") +} diff --git a/pkg/registry/file/sqliteobject_testenv_test.go b/pkg/registry/file/sqliteobject_testenv_test.go new file mode 100644 index 000000000..bdf2c9222 --- /dev/null +++ b/pkg/registry/file/sqliteobject_testenv_test.go @@ -0,0 +1,459 @@ +package file + +// Shared test environment for the ObjectStore (SQLite-native ContainerProfile +// backend) tests: one pool, the ObjectStore, a legacy StorageImpl over the SAME +// pool carrying the kind-ownership guard (the production wiring under +// config.ContainerProfileSqliteBackend), a real ContainerProfileProcessor, and +// a per-connection statement authorizer that records every table an +// operation touched — the instrument INV-1 and INV-4 assert on. + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "reflect" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" + + helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/kubescape/storage/pkg/apis/softwarecomposition/install" + "github.com/kubescape/storage/pkg/config" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apiserver/pkg/storage" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitemigration" + "zombiezen.com/go/sqlite/sqlitex" +) + +const testCPPrefix = "/spdx.softwarecomposition.kubescape.io/containerprofile/" + +// tableAction is one authorizer observation: an SQLite action type on a table +// (SQLITE_READ / INSERT / UPDATE / DELETE), recorded per connection. +type tableAction struct { + op sqlite.OpType + table string + conn *sqlite.Conn + // txn is set for BEGIN/COMMIT/ROLLBACK observations (Operation()). + txn string +} + +// tableRecorder is installed through PoolOptions.Authorizer on every +// connection; it records every table-touching action from connection +// creation on and is never windowed off: a statement first prepared outside +// a recording window re-executes later through the connection's statement +// cache with no authorizer call, so a windowed recorder would miss exactly +// the statements prepared during setup. Tests take a mark and read what came +// after it. Safe from any goroutine. +type tableRecorder struct { + mu sync.Mutex + actions []tableAction +} + +func (r *tableRecorder) authorizer(conn *sqlite.Conn) sqlite.Authorizer { + return sqlite.AuthorizeFunc(func(a sqlite.Action) sqlite.AuthResult { + switch a.Type() { + case sqlite.OpRead, sqlite.OpInsert, sqlite.OpUpdate, sqlite.OpDelete: + if t := a.Table(); t != "" { + r.mu.Lock() + r.actions = append(r.actions, tableAction{op: a.Type(), table: t, conn: conn}) + r.mu.Unlock() + } + case sqlite.OpTransaction: + r.mu.Lock() + r.actions = append(r.actions, tableAction{op: a.Type(), conn: conn, txn: a.Operation()}) + r.mu.Unlock() + } + return sqlite.AuthResultOK + }) +} + +// mark returns the current position; since returns everything recorded +// after a mark. +func (r *tableRecorder) mark() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.actions) +} + +func (r *tableRecorder) since(mark int) []tableAction { + r.mu.Lock() + defer r.mu.Unlock() + return append([]tableAction(nil), r.actions[mark:]...) +} + +func isWriteOp(op sqlite.OpType) bool { + return op == sqlite.OpInsert || op == sqlite.OpUpdate || op == sqlite.OpDelete +} + +type objectStoreEnv struct { + t *testing.T + ctx context.Context + dir string + dbPath string + pool *sqlitemigration.Pool + gate *writeGate + store *ObjectStore + cp *objectStoreCPStorage + legacy *StorageImpl + legacyFs afero.Fs + processor *ContainerProfileProcessor + wd *WatchDispatcher + scheme *runtime.Scheme + rec *tableRecorder + // fixture is the non-pool handle tests seed state through (CR-2b). + fixture *sqlite.Conn + tpl softwarecomposition.ContainerProfile + baseNm string + ns string + baseKey string + now time.Time +} + +type envOption func(*envConfig) + +type envConfig struct { + poolSize int + checkpointBytes int64 + checkpointEvery time.Duration + autoCheckpoint bool +} + +func withPoolSize(n int) envOption { return func(c *envConfig) { c.poolSize = n } } +func withCheckpoint(bytes int64, every time.Duration) envOption { + return func(c *envConfig) { c.checkpointBytes = bytes; c.checkpointEvery = every } +} + +// newObjectStoreEnv builds the environment. singleWriterEnabled stays in its +// default (true) position: the legacy instance is the flag-default reference. +func newObjectStoreEnv(t *testing.T, opts ...envOption) *objectStoreEnv { + t.Helper() + cfg := envConfig{poolSize: DefaultPoolSize, checkpointEvery: time.Hour} + for _, o := range opts { + o(&cfg) + } + dir := t.TempDir() + dbPath := filepath.Join(dir, "metadata.sq3") + rec := &tableRecorder{} + pool := NewPoolWithOptions(dbPath, PoolOptions{ + Size: cfg.poolSize, + BusyTimeout: 5 * time.Second, + DisableAutoCheckpoint: !cfg.autoCheckpoint, + Authorizer: rec.authorizer, + }) + // AC-G1, registered first so it runs after the store and pool closed. + armUngatedWriteCheck(t, pool) + sch := runtime.NewScheme() + install.Install(sch) + wd := NewWatchDispatcher() + + legacyFs := afero.NewMemMapFs() + legacy := NewStorageImpl(legacyFs, DefaultStorageRoot, pool, wd, sch).(*StorageImpl) + legacy.SetForeignKinds(IsContainerProfileKind) + + processor := NewContainerProfileProcessor(config.Config{DefaultNamespace: "kubescape", MaxContainerProfileSize: 40000}, nil) + processor.Interval = 0 + processor.Workers = 1 + processor.DeleteThreshold = 24 * time.Hour + + gateCtx, gateCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer gateCancel() + gate, err := newWriteGate(gateCtx, pool) + require.NoError(t, err) + legacy.SetWriteGate(gate) + store, err := NewObjectStore(pool, dbPath, wd, sch, processor, legacy, gate, ObjectStoreOptions{ + CheckpointThresholdBytes: cfg.checkpointBytes, + CheckpointInterval: cfg.checkpointEvery, + }) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, store.Close()) + require.NoError(t, gate.Close()) + closed := make(chan error, 1) + go func() { closed <- pool.Close() }() + select { + case err := <-closed: + require.NoError(t, err) + case <-time.After(10 * time.Second): + t.Errorf("pool.Close did not return: a connection was never returned (K-5)") + } + }) + + content, err := os.ReadFile("testdata/p1.json") + require.NoError(t, err) + var tpl softwarecomposition.ContainerProfile + require.NoError(t, json.Unmarshal(content, &tpl)) + baseNm, _ := SplitProfileName(tpl.Name) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + t.Cleanup(cancel) + return &objectStoreEnv{ + t: t, ctx: ctx, dir: dir, dbPath: dbPath, pool: pool, gate: gate, store: store, + cp: processor.ContainerProfileStorage.(*objectStoreCPStorage), + legacy: legacy, legacyFs: legacyFs, processor: processor, wd: wd, scheme: sch, rec: rec, + fixture: openFixtureConn(t, pool, dbPath, 5*time.Second), + tpl: tpl, baseNm: baseNm, ns: tpl.Namespace, + baseKey: testCPPrefix + tpl.Namespace + "/" + baseNm, + now: time.Now().Round(0), + } +} + +func (e *objectStoreEnv) tsKey(suffix string) string { return e.baseKey + "-" + suffix } + +func (e *objectStoreEnv) reportTimestamp(n int) string { + return e.now.Add(time.Duration(n-10) * time.Minute).String() +} + +// ts clones the template as report n of its series under suffix, chained to +// report n-1 so consolidation sees one continuous series. +func (e *objectStoreEnv) ts(suffix string, n int, status, completion string) *softwarecomposition.ContainerProfile { + p := e.tpl.DeepCopy() + p.Name = e.baseNm + "-" + suffix + p.ResourceVersion = "" + p.UID = uuid.NewUUID() + prev := "0001-01-01 00:00:00 +0000 UTC" + if n > 1 { + prev = e.reportTimestamp(n - 1) + } + p.Annotations[helpersv1.ReportTimestampMetadataKey] = e.reportTimestamp(n) + p.Annotations[helpersv1.PreviousReportTimestampMetadataKey] = prev + p.Annotations[helpersv1.StatusMetadataKey] = status + p.Annotations[helpersv1.CompletionMetadataKey] = completion + return p +} + +// plain returns a non-TS profile named name in the template's namespace. +func (e *objectStoreEnv) plain(name string) *softwarecomposition.ContainerProfile { + p := e.tpl.DeepCopy() + p.Name = name + p.ResourceVersion = "" + p.UID = uuid.NewUUID() + delete(p.Annotations, helpersv1.ReportSeriesIdMetadataKey) + return p +} + +func (e *objectStoreEnv) key(name string) string { return testCPPrefix + e.ns + "/" + name } + +func (e *objectStoreEnv) create(p *softwarecomposition.ContainerProfile) *softwarecomposition.ContainerProfile { + e.t.Helper() + out := &softwarecomposition.ContainerProfile{} + require.NoError(e.t, e.store.Create(e.ctx, e.key(p.Name), p, out, 0)) + return out +} + +func (e *objectStoreEnv) get(key string) (*softwarecomposition.ContainerProfile, error) { + out := &softwarecomposition.ContainerProfile{} + err := e.store.Get(e.ctx, key, storage.GetOptions{}, out) + return out, err +} + +func (e *objectStoreEnv) mustGet(key string) *softwarecomposition.ContainerProfile { + e.t.Helper() + out, err := e.get(key) + require.NoError(e.t, err) + return out +} + +func (e *objectStoreEnv) tick() { + e.t.Helper() + require.NoError(e.t, e.processor.ConsolidateTimeSeries(e.ctx)) +} + +// withConn runs fn on a pool connection: reads only. A write here would be +// an ungated write under AC-G1; seed state through withFixture instead. +func (e *objectStoreEnv) withConn(fn func(conn *sqlite.Conn)) { + e.t.Helper() + conn, err := e.pool.Take(e.ctx) + require.NoError(e.t, err) + defer e.pool.Put(conn) + fn(conn) +} + +// withFixture runs fn on the non-pool fixture handle (CR-2b). +func (e *objectStoreEnv) withFixture(fn func(conn *sqlite.Conn)) { + e.t.Helper() + fn(e.fixture) +} + +// dbRow is what the tables hold for one key, as seen from a fresh connection. +type dbRow struct { + metaExists bool + payloadExists bool + rv *int64 + uid *string + jsonRV string + jsonUID string + tsRows int +} + +// inspect reads the INV-2 view of key from its own connection (a snapshot no +// in-flight transaction can influence). +func (e *objectStoreEnv) inspect(key string) dbRow { + e.t.Helper() + var row dbRow + e.withConn(func(conn *sqlite.Conn) { + row = inspectRow(e.t, conn, key) + }) + return row +} + +func inspectRow(t *testing.T, conn *sqlite.Conn, key string) dbRow { + t.Helper() + _, _, kind, _, ns, name := K8sPathToKeys(key) + var row dbRow + require.NoError(t, sqlitex.Execute(conn, + `SELECT rv, uid, json_extract(CAST(metadata AS TEXT), '$.resourceVersion'), json_extract(CAST(metadata AS TEXT), '$.uid') + FROM metadata WHERE kind = ? AND namespace = ? AND name = ?`, + &sqlitex.ExecOptions{Args: []any{kind, ns, name}, ResultFunc: func(stmt *sqlite.Stmt) error { + row.metaExists = true + if stmt.ColumnType(0) != sqlite.TypeNull { + v := stmt.ColumnInt64(0) + row.rv = &v + } + if stmt.ColumnType(1) != sqlite.TypeNull { + v := stmt.ColumnText(1) + row.uid = &v + } + row.jsonRV = stmt.ColumnText(2) + row.jsonUID = stmt.ColumnText(3) + return nil + }})) + require.NoError(t, sqlitex.Execute(conn, + `SELECT 1 FROM payloads WHERE kind = ? AND namespace = ? AND name = ?`, + &sqlitex.ExecOptions{Args: []any{kind, ns, name}, ResultFunc: func(*sqlite.Stmt) error { row.payloadExists = true; return nil }})) + require.NoError(t, sqlitex.Execute(conn, + `SELECT count(*) FROM time_series WHERE kind = ? AND namespace = ? AND name = ?`, + &sqlitex.ExecOptions{Args: []any{NormalizeContainerProfileKind(kind), ns, name}, ResultFunc: func(stmt *sqlite.Stmt) error { row.tsRows = int(stmt.ColumnInt64(0)); return nil }})) + return row +} + +// assertINV2 asserts the design's INV-2 for key on a fresh connection: +// metadata row ⇔ payloads row; rv == json resourceVersion; uid == json uid. +func assertINV2(t *testing.T, conn *sqlite.Conn, key string) { + t.Helper() + row := inspectRow(t, conn, key) + require.Equal(t, row.metaExists, row.payloadExists, "INV-2: metadata row (%v) must exist iff payloads row (%v) for %s", row.metaExists, row.payloadExists, key) + if !row.metaExists { + return + } + require.NotNil(t, row.rv, "INV-2: rv column NULL for %s", key) + require.NotNil(t, row.uid, "INV-2: uid column NULL for %s", key) + require.Equal(t, strings.TrimSpace(row.jsonRV), strconv.FormatInt(*row.rv, 10), "INV-2: rv column != json resourceVersion for %s", key) + require.Equal(t, row.jsonUID, *row.uid, "INV-2: uid column != json uid for %s", key) +} + +// allCPKeys lists every containerprofile key present in metadata or payloads. +func allCPKeys(t *testing.T, conn *sqlite.Conn) []string { + t.Helper() + seen := map[string]bool{} + var keys []string + add := func(stmt *sqlite.Stmt) error { + k := testCPPrefix + stmt.ColumnText(0) + "/" + stmt.ColumnText(1) + if !seen[k] { + seen[k] = true + keys = append(keys, k) + } + return nil + } + require.NoError(t, sqlitex.Execute(conn, `SELECT namespace, name FROM metadata WHERE kind = 'containerprofile'`, &sqlitex.ExecOptions{ResultFunc: add})) + require.NoError(t, sqlitex.Execute(conn, `SELECT namespace, name FROM payloads WHERE kind = 'containerprofile'`, &sqlitex.ExecOptions{ResultFunc: add})) + return keys +} + +// canonicalCP returns a copy of cp normalised for cross-codec comparison: the +// two documented codec deltas are removed (creationTimestamp truncated to +// seconds in UTC; empty collections nilified) plus raw header JSON compacted +// (gob keeps the bytes verbatim, encoding/json compacts RawMessage on output). +// Everything else must match exactly. +func canonicalCP(cp *softwarecomposition.ContainerProfile) *softwarecomposition.ContainerProfile { + out := cp.DeepCopy() + if !out.CreationTimestamp.IsZero() { + out.CreationTimestamp = metav1.NewTime(out.CreationTimestamp.Time.Truncate(time.Second).UTC()) + } + // Header values are emitted by the endpoint analyzer in map-iteration + // order (nondeterministic on either backend), so canonicalise them: parse, + // sort each header's values, re-marshal (sorted keys, compact). + compactHeaders := func(eps []softwarecomposition.HTTPEndpoint) { + for i := range eps { + if len(eps[i].Headers) == 0 { + continue + } + var hdr map[string][]string + if err := json.Unmarshal(eps[i].Headers, &hdr); err != nil { + var buf bytes.Buffer + if err := json.Compact(&buf, eps[i].Headers); err == nil { + eps[i].Headers = buf.Bytes() + } + continue + } + for k := range hdr { + sort.Strings(hdr[k]) + } + if b, err := json.Marshal(hdr); err == nil { + eps[i].Headers = b + } + } + } + compactHeaders(out.Spec.Endpoints) + for i := range out.Spec.Containers { + compactHeaders(out.Spec.Containers[i].Endpoints) + } + for i := range out.Spec.InitContainers { + compactHeaders(out.Spec.InitContainers[i].Endpoints) + } + for i := range out.Spec.EphemeralContainers { + compactHeaders(out.Spec.EphemeralContainers[i].Endpoints) + } + nilifyEmpty(reflect.ValueOf(out).Elem()) + return out +} + +// nilifyEmpty sets every zero-length slice or map reachable from v to nil. +func nilifyEmpty(v reflect.Value) { + switch v.Kind() { + case reflect.Pointer, reflect.Interface: + if !v.IsNil() { + nilifyEmpty(v.Elem()) + } + case reflect.Struct: + for i := 0; i < v.NumField(); i++ { + if v.Field(i).CanSet() { + nilifyEmpty(v.Field(i)) + } + } + case reflect.Slice: + if v.Len() == 0 { + if v.CanSet() { + v.Set(reflect.Zero(v.Type())) + } + return + } + for i := 0; i < v.Len(); i++ { + nilifyEmpty(v.Index(i)) + } + case reflect.Map: + if v.Len() == 0 && v.CanSet() { + v.Set(reflect.Zero(v.Type())) + return + } + for _, k := range v.MapKeys() { + mv := v.MapIndex(k) + if mv.Kind() == reflect.Struct || mv.Kind() == reflect.Pointer { + cp := reflect.New(mv.Type()).Elem() + cp.Set(mv) + nilifyEmpty(cp) + v.SetMapIndex(k, cp) + } + } + } +} diff --git a/pkg/registry/file/storage.go b/pkg/registry/file/storage.go index 4caee51f9..0d5ecf8a7 100644 --- a/pkg/registry/file/storage.go +++ b/pkg/registry/file/storage.go @@ -12,6 +12,7 @@ import ( "os/exec" "path/filepath" "reflect" + "strconv" "strings" "sync" "time" @@ -32,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/conversion" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/storage" "zombiezen.com/go/sqlite" @@ -56,8 +58,47 @@ var ( // base ContainerProfile is already Completed/Full: nothing updates such a // profile (softwarecomposition.IsCompletedFull). ErrProfileFrozen = errors.New("profile is completed/full and cannot be updated") + // ErrGateRequiresSingleWriter is returned by Create/GuaranteedUpdate on a + // StorageImpl that shares the write gate while singleWriterEnabled is + // false: that combination would route every REST write through + // CreateWithConn/GuaranteedUpdateWithConn holding a pool connection while + // queued on the gate (write-gate-sharing §3.3). main.go refuses the + // configuration at startup; this catches a test flipping the package + // variable directly. + ErrGateRequiresSingleWriter = errors.New("storage: the shared write gate requires the single-writer path (singleWriterEnabled=false is refused)") ) +// gatedWrite is the one helper every legacy write site runs its SQL through +// (write-gate-sharing §3.1). With no gate it is today's code, byte for byte: +// bare autocommit statements (txn=false, W3–W9) or the savepoint the site +// has today (txn=true, W1/W2) on the caller's connection. With a gate, both +// are BEGIN IMMEDIATE … COMMIT on the gate's connection and conn is ignored; +// fn receives the gate-marked ctx and must thread it into anything it calls +// (INV-1′: no lock, no pool take, no processor, no dispatch, no gob, no exec; +// one same-directory rename of an already-written file is the one exemption). +func gatedWrite(gate *writeGate, ctx context.Context, conn *sqlite.Conn, priority writePriority, path, kind string, txn bool, fn func(ctx context.Context, conn *sqlite.Conn) error) error { + if gate == nil { + if !txn { + return fn(ctx, conn) + } + release := sqlitex.Save(conn) + err := fn(ctx, conn) + release(&err) + return err + } + return gate.run(ctx, priority, path, kind, fn) +} + +// SetWriteGate hands the StorageImpl the process's shared write gate (nil = +// legacy behaviour, no gate). Called once at wiring time, before traffic. +func (s *StorageImpl) SetWriteGate(gate *WriteGate) { + s.gate = gate +} + +func (s *StorageImpl) write(ctx context.Context, conn *sqlite.Conn, priority writePriority, path, kind string, txn bool, fn func(ctx context.Context, conn *sqlite.Conn) error) error { + return gatedWrite(s.gate, ctx, conn, priority, path, kind, txn, fn) +} + // lockTimeout is the hardcoded backstop for lock acquisition. It sits well under the // ~60s outer apiserver request deadline so a contended request fails fast with a // Retry-After signal instead of hanging to the full request timeout. @@ -279,6 +320,56 @@ type StorageImpl struct { // tests exercising unrelated paths) never spins up an unused goroutine. writer *singleWriter writerOnce sync.Once + + // foreignKinds is the kind-ownership guard (design §5.6, INV-4): when set, + // every full-object operation on a key whose kind segment it reports as + // foreign is REFUSED with an InternalError instead of touching the shared + // metadata row, the payload file or the time_series table. Set from + // config.ContainerProfileSqliteBackend to IsContainerProfileKind, so a + // mis-wired legacy path is a loud failure, never a silent self-repair + // delete of a row the ObjectStore owns. Metadata-only reads (the shared + // row both backends agree on) are not refused. nil = no guard. + foreignKinds func(kind string) bool + + // gate is the process's shared write gate (write-gate-sharing §3): when + // set, every write statement of this instance runs on the gate's + // connection through write(); nil = today's code on pool connections. + gate *writeGate +} + +// SetForeignKinds installs the kind-ownership guard; nil removes it. +func (s *StorageImpl) SetForeignKinds(f func(kind string) bool) { + s.foreignKinds = f +} + +// kindOfKey returns the kind segment of a storage key ("/prefix/root/kind/…") +// or of getListWithSpec's relative "apiVersion/kind/…" path. +func kindOfKey(key string) string { + if strings.HasPrefix(key, "/") { + _, _, kind, _, _, _ := K8sPathToKeys(key) + return kind + } + parts := strings.Split(key, "/") + if len(parts) >= 2 { + return parts[1] + } + return "" +} + +// refuseForeign returns the guard's InternalError when key's kind is foreign +// to this instance, counting and logging the refusal. +func (s *StorageImpl) refuseForeign(op, key string) error { + if s.foreignKinds == nil { + return nil + } + kind := kindOfKey(key) + if !s.foreignKinds(kind) { + return nil + } + metrics.IncCPOwnershipRefusal(op) + logger.L().Error("legacy StorageImpl refused an operation on a kind owned by another backend", + helpers.String("op", op), helpers.String("kind", kind), helpers.String("key", key)) + return apierrors.NewInternalError(fmt.Errorf("storage: key %q has kind %q, which is owned by the ContainerProfile SQLite backend; the legacy file store refuses %s", key, kind, op)) } func (s *StorageImpl) EnableResourceSizeEstimation(keysFunc storage.KeysFunc) error { @@ -289,11 +380,7 @@ func (s *StorageImpl) EnableResourceSizeEstimation(keysFunc storage.KeysFunc) er // and transparently retries without it when the filesystem returns an // "unsupported" error (e.g. EINVAL on tmpfs/overlayfs). func (s *StorageImpl) openPayloadFileWithFallback(path string, flag int, perm os.FileMode) (afero.File, error) { - f, err := s.appFs.OpenFile(path, openFlagDirect|flag, perm) - if err != nil && isDirectIOUnsupported(err) { - f, err = s.appFs.OpenFile(path, flag, perm) - } - return f, err + return openPayloadFileWithFallbackFs(s.appFs, path, flag, perm) } func (s *StorageImpl) Stats(_ context.Context) (storage.Stats, error) { @@ -380,6 +467,25 @@ func IsPayloadFile(path string) bool { return strings.HasSuffix(path, GobExt) } +// keyFromPayloadPath recovers the storage key a payload file's path was +// written to by makePayloadPath(root, key) (i.e. filepath.Join(root, key) + +// GobExt), given only the file's actual on-disk path and the root it was +// walked under. Reverses via filepath.Rel rather than slicing path at +// len(root): root's exact textual form (whether "/data", "/data/", ".", or +// "/") does not survive into path, which afero.Walk always reports as the +// filepath.Join'd, cleaned form -- slicing by byte length silently +// mis-derives the key whenever root's length doesn't match what Join +// actually consumed (a trailing slash overcounts by one; root "/" or "." +// each have their own off-by-one). Rel is exact for every root shape. +func keyFromPayloadPath(root, path string) (string, error) { + rel, err := filepath.Rel(root, path) + if err != nil { + return "", fmt.Errorf("relativize %s under %s: %w", path, root, err) + } + rel = strings.TrimSuffix(rel, GobExt) + return "/" + filepath.ToSlash(rel), nil +} + func poolContext() (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), poolTimeout) } @@ -400,7 +506,12 @@ func (s *StorageImpl) keyFromPath(path string) string { // for callers to use as the lightweight watch-event object: watchers created // without ResourceVersionFullSpec receive this reduced object instead of the // full one, to avoid bloating watch traffic with large Spec payloads. -func (s *StorageImpl) saveObject(conn *sqlite.Conn, key string, obj runtime.Object, metaOut runtime.Object, checksum string) (runtime.Object, error) { +// +// The metadata row and the payload rename commit together through write() +// (W2 of write-gate-sharing §3.2): today's savepoint on conn with no gate, +// the gate's transaction with one. priority and path label the hold — +// legacy_commit from the REST paths, migrate from the gob-migration rewrites. +func (s *StorageImpl) saveObject(ctx context.Context, conn *sqlite.Conn, key string, obj runtime.Object, metaOut runtime.Object, checksum string, priority writePriority, path string) (runtime.Object, error) { // increment resourceVersion if version, err := s.versioner.ObjectResourceVersion(obj); err == nil { if err := s.versioner.UpdateObject(obj, version+1); err != nil { @@ -474,17 +585,18 @@ func (s *StorageImpl) saveObject(conn *sqlite.Conn, key string, obj runtime.Obje if renamePayload == nil { renamePayload = s.appFs.Rename } - release := sqlitex.Save(conn) - err = func() error { + observeStmt("Save:saveObject") + err = s.write(ctx, conn, priority, path, resourceFromKey(key), true, func(_ context.Context, conn *sqlite.Conn) error { if werr := writeMeta(conn, key, metadata); werr != nil { return fmt.Errorf("write metadata: %w", werr) } + renameStart := time.Now() if rerr := renamePayload(tmpPayloadPath, finalPayloadPath); rerr != nil { return fmt.Errorf("rename payload into place: %w", rerr) } + metrics.ObserveSqliteWriteHoldStep(path, "rename", time.Since(renameStart)) return nil - }() - release(&err) + }) if err != nil { _ = s.appFs.Remove(tmpPayloadPath) return nil, err @@ -519,6 +631,12 @@ func (s *StorageImpl) Create(ctx context.Context, key string, obj, metaOut runti if singleWriterEnabled { return s.createSingleWriter(ctx, key, obj, metaOut, priorityHigh) } + if s.gate != nil { + if err := s.refuseForeign("create", key); err != nil { + return err + } + return ErrGateRequiresSingleWriter + } poolCtx, cancel := poolContext() defer cancel() beforePool := time.Now() @@ -533,6 +651,9 @@ func (s *StorageImpl) Create(ctx context.Context, key string, obj, metaOut runti } func (s *StorageImpl) CreateWithConn(ctx context.Context, conn *sqlite.Conn, key string, obj, metaOut runtime.Object, _ uint64) error { + if err := s.refuseForeign("create", key); err != nil { + return err + } ctx, span := otel.Tracer("").Start(ctx, "StorageImpl.Create") span.SetAttributes(attribute.String("key", key)) defer span.End() @@ -556,6 +677,18 @@ func (s *StorageImpl) CreateWithConn(ctx context.Context, conn *sqlite.Conn, key if _, err := s.appFs.Stat(makePayloadPath(filepath.Join(s.root, key))); err == nil { return storage.NewKeyExistsError(key, 0) } + // Flag-off objects can exist solely in SQLite. Check under the key lock + // before a legacy write can replace their metadata; the single-writer + // Create path performs its own commit-time database check. + visible, err := isFallbackVisible(conn, key) + if err != nil { + // Fail closed: never overwrite a key we could not check. + logger.L().Ctx(ctx).Error("Create - fallback existence check failed", helpers.Error(err), helpers.String("key", key)) + return fmt.Errorf("create existence check: %w", err) + } + if visible { + return storage.NewKeyExistsError(key, 0) + } // resourceVersion should not be set on create if version, err := s.versioner.ObjectResourceVersion(obj); err == nil && version != 0 { msg := "resourceVersion should not be set on objects to be created" @@ -584,7 +717,7 @@ func (s *StorageImpl) CreateWithConn(ctx context.Context, conn *sqlite.Conn, key } } // save object - metaEvent, err := s.saveObject(conn, key, obj, metaOut, "") + metaEvent, err := s.saveObject(ctx, conn, key, obj, metaOut, "", priorityHigh, holdPathLegacyCommit) if err != nil { logger.L().Ctx(ctx).Error("Create - save object failed", helpers.Error(err), helpers.String("key", key)) return err @@ -609,6 +742,18 @@ func (s *StorageImpl) CreateWithConn(ctx context.Context, conn *sqlite.Conn, key // callers (none currently, but its signature/behavior is preserved as // documented Phase 1 scope). func (s *StorageImpl) Delete(ctx context.Context, key string, metaOut runtime.Object, _ *storage.Preconditions, _ storage.ValidateObjectFunc, _ runtime.Object, _ storage.DeleteOptions) error { + if s.gate != nil { + // The statements run on the gate's connection: the per-key lock only, + // no pool connection held while queued (write-gate-sharing §3.2/§3.3). + if err := s.lockKey(ctx, "delete", key, s.locks.Lock); err != nil { + return err + } + defer s.locks.Unlock(key) + ctx, span := otel.Tracer("").Start(ctx, "StorageImpl.Delete") + span.SetAttributes(attribute.String("key", key)) + defer span.End() + return s.delete(ctx, nil, key, metaOut, nil, nil, nil, storage.DeleteOptions{}) + } conn, err := s.acquireLockedConn(ctx, "delete", key, s.locks.Lock, s.locks.Unlock) if err != nil { return err @@ -622,6 +767,28 @@ func (s *StorageImpl) Delete(ctx context.Context, key string, metaOut runtime.Ob return s.delete(ctx, conn, key, metaOut, nil, nil, nil, storage.DeleteOptions{}) } +// lockKey is acquireLockedConn's lock step alone, for the gated paths that +// need no pool connection: acquire under lockTimeout with the same metrics +// and the same fail-fast error. +func (s *StorageImpl) lockKey(ctx context.Context, op, key string, acquire func(context.Context, string) error) error { + _, spanLock := otel.Tracer("").Start(ctx, "waiting for lock") + beforeLock := time.Now() + lockCtx, lockCancel := context.WithTimeout(ctx, lockTimeout) + err := acquire(lockCtx, key) + lockCancel() + spanLock.End() + lockDuration := time.Since(beforeLock) + if err != nil { + metrics.ObserveLockWait(resourceFromKey(key), metrics.OutcomeTimeout, lockDuration) + return newContentionTimeoutError(op, key, err) + } + metrics.ObserveLockWait(resourceFromKey(key), metrics.OutcomeAcquired, lockDuration) + if lockDuration > time.Second { + logger.L().Debug(op, helpers.String("key", key), helpers.String("lockDuration", lockDuration.String())) + } + return nil +} + func (s *StorageImpl) DeleteWithConn(ctx context.Context, conn *sqlite.Conn, key string, metaOut runtime.Object, _ *storage.Preconditions, _ storage.ValidateObjectFunc, _ runtime.Object, _ storage.DeleteOptions) error { ctx, span := otel.Tracer("").Start(ctx, "StorageImpl.Delete") span.SetAttributes(attribute.String("key", key)) @@ -651,6 +818,14 @@ func (s *StorageImpl) DeleteWithConn(ctx context.Context, conn *sqlite.Conn, key // releasing whatever they hold (shard turn, connection, per-key lock). func (s *StorageImpl) deleteLocked(ctx context.Context, conn *sqlite.Conn, key string, metaOut runtime.Object) error { p := filepath.Join(s.root, key) + if s.gate != nil { + return s.deleteLockedGated(ctx, key, metaOut, p) + } + // Delete the SQL payload before metadata. On failure preserve the other + // state so the caller can retry the complete deletion. + if err := DeletePayloads(conn, key); err != nil { + return fmt.Errorf("delete payloads: %w", err) + } // delete metadata in SQLite err := DeleteMetadata(conn, key, metaOut) if err != nil { @@ -671,7 +846,48 @@ func (s *StorageImpl) deleteLocked(ctx context.Context, conn *sqlite.Conn, key s return nil } +// deleteLockedGated is deleteLocked under the shared gate (W3): the row +// delete (and the time_series delete for a containerprofile) in one gated +// transaction with the deleted row's JSON captured raw; the payload Remove +// and the decode into metaOut after release (INV-1′). conn is not used: the +// caller may hold none (Delete) or one it keeps for reads (DeleteWithConn). +func (s *StorageImpl) deleteLockedGated(ctx context.Context, key string, metaOut runtime.Object, p string) error { + _, _, kind, _, _, _ := K8sPathToKeys(key) + var raw []byte + err := s.write(ctx, nil, priorityHigh, holdPathLegacyDelete, resourceFromKey(key), false, func(_ context.Context, conn *sqlite.Conn) error { + var derr error + raw, derr = deleteMetadataRaw(conn, key) + if derr != nil { + return derr + } + if IsContainerProfileKind(kind) { + if terr := DeleteTimeSeriesContainerEntries(conn, key); terr != nil { + return fmt.Errorf("delete time series entries: %w", terr) + } + } + return nil + }) + if err != nil { + logger.L().Ctx(ctx).Error("Delete - delete metadata failed", helpers.Error(err), helpers.String("key", key)) + } + if rerr := s.appFs.Remove(makePayloadPath(p)); rerr != nil { + logger.L().Ctx(ctx).Error("Delete - remove json file failed", helpers.Error(rerr), helpers.String("key", key)) + } + if err != nil { + return err + } + if metaOut != nil && len(raw) > 0 { + if uerr := json.Unmarshal(raw, metaOut); uerr != nil { + return fmt.Errorf("delete metadata: %w", uerr) + } + } + return nil +} + func (s *StorageImpl) delete(ctx context.Context, conn *sqlite.Conn, key string, metaOut runtime.Object, _ *storage.Preconditions, _ storage.ValidateObjectFunc, _ runtime.Object, _ storage.DeleteOptions) error { + if err := s.refuseForeign("delete", key); err != nil { + return err + } if err := s.deleteLocked(ctx, conn, key, metaOut); err != nil { return err } @@ -782,6 +998,13 @@ func (s *StorageImpl) get(ctx context.Context, conn *sqlite.Conn, key string, op return json.Unmarshal(metadata, objPtr) } + // The metadata-only branch above reads the shared row both backends agree + // on; everything below touches the payload file and the self-repair + // deletes, which a foreign kind's owner must never see (INV-4). + if err := s.refuseForeign("get", key); err != nil { + return err + } + // noLock callers perform unsynchronized file I/O by default. Acquire a // temporary read lock so that a concurrent saveObject cannot truncate or // overwrite the file while we are decoding it. The lock is released via @@ -810,7 +1033,22 @@ func (s *StorageImpl) get(ctx context.Context, conn *sqlite.Conn, key string, op // SQLite's write lock for the busy timeout, so a read of an absent // key must not issue it. if _, rerr := ReadMetadata(conn, key); rerr == nil { - _ = DeleteMetadata(conn, key, nil) + outcome, inspectionErr := s.inspectPayloadsFallback(conn, key, objPtr) + if inspectionErr != nil { + // An unreadable object is not absent. In particular, + // IgnoreNotFound must not turn this into an empty object. + return fmt.Errorf("get payloads fallback: %w", inspectionErr) + } + switch outcome { + case fallbackServed: + return nil + case fallbackPrunable: + _ = s.repairDeleteWithPayloads(ctx, conn, key) + case fallbackIneligible: + _ = s.repairDelete(ctx, conn, key) + } + } else if !errors.Is(rerr, ErrMetadataNotFound) { + return fmt.Errorf("get fallback metadata: %w", rerr) } if opts.IgnoreNotFound { return runtime.SetZeroValue(objPtr) @@ -885,7 +1123,7 @@ func (s *StorageImpl) get(ctx context.Context, conn *sqlite.Conn, key string, op if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { // irrecoverable error, delete corresponding data - _ = DeleteMetadata(conn, key, nil) + _ = s.repairDelete(ctx, conn, key) _ = s.appFs.Remove(makePayloadPath(p)) logger.L().Ctx(ctx).Error("Get - gob error, treating as corrupted and removing files", helpers.Error(err), helpers.String("key", key)) if opts.IgnoreNotFound { @@ -898,6 +1136,102 @@ func (s *StorageImpl) get(ctx context.Context, conn *sqlite.Conn, key string, op return err } +// fallbackOutcome distinguishes a served object from the two read-repair +// cases. Inspection failures are returned separately and prohibit mutation. +type fallbackOutcome int + +const ( + fallbackIneligible fallbackOutcome = iota // Preserve any recoverable payload. + fallbackServed + fallbackPrunable // A non-time-series legacy write superseded the payload. +) + +// inspectPayloadsFallback is called only when the legacy file is missing. +// It serves a joined metadata/payloads row with non-NULL rv and no time-series +// marker. Query, decode, and conversion failures leave both rows untouched. +func (s *StorageImpl) inspectPayloadsFallback(conn *sqlite.Conn, key string, objPtr runtime.Object) (fallbackOutcome, error) { + candidate, err := readFallbackCandidate(conn, key) + if err != nil { + return fallbackIneligible, err + } + if candidate == nil || candidate.isTimeSeries { + return fallbackIneligible, nil + } + // Every legacy write clears rv. Its old SQL payload must not be served + // after a lost file rename, and can be pruned at this missing-file site. + if candidate.rvNull { + return fallbackPrunable, nil + } + if !candidate.payloadsFound { + return fallbackIneligible, nil + } + if s.scheme == nil { + return fallbackIneligible, fmt.Errorf("no scheme to decode payloads body") + } + if err := decodePayloadBody(s.scheme, candidate.encoding, candidate.body, objPtr); err != nil { + return fallbackIneligible, fmt.Errorf("decode payloads body: %w", err) + } + accessor, err := meta.Accessor(objPtr) + if err != nil { + return fallbackIneligible, fmt.Errorf("access payload metadata: %w", err) + } + // Use the authoritative columns, as cpexport does. + accessor.SetResourceVersion(strconv.FormatInt(candidate.rv, 10)) + if candidate.uid != "" { + accessor.SetUID(types.UID(candidate.uid)) + } + logger.L().Debug("Get - payload file missing, served from the payloads row", helpers.String("key", key)) + return fallbackServed, nil +} + +// isFallbackVisible checks the database fallback predicate without decoding. +// Even an eligible payload this binary cannot decode must block Create. +// Query errors are returned so callers cannot overwrite uninspected state. +func isFallbackVisible(conn *sqlite.Conn, key string) (bool, error) { + candidate, err := readFallbackCandidate(conn, key) + if err != nil { + return false, err + } + if candidate == nil { + return false, nil + } + return !candidate.rvNull && !candidate.isTimeSeries && candidate.payloadsFound, nil +} + +// repairDelete is get()'s self-repair DELETE of a metadata row whose payload +// is missing, empty or unmigratable (W4/W5/W6a/W7a): a low-priority write on +// the repair path. The caller holds the key's read (or write) lock and its +// pool connection while queued — accepted for these cold paths +// (write-gate-sharing §3.3). The error is the caller's to swallow, as today. +// +// It deletes the metadata row and NOTHING else. In particular it must never +// touch the key's payloads row: this function is shared by all four of get()'s +// self-repair sites, three of which fire on an undecodable .g FILE (gob EOF, +// and the two migration-tool failures) where a perfectly valid native payload +// may sit next to the corrupt file. Those three sites are explicitly out of +// scope for the rollback-safety guard and must keep leaving a recoverable +// orphan payload behind, exactly as they did before it +// (docs/features/containerprofile-sqlite-backend.md). Only get()'s +// missing-file site prunes payloads, and only in the narrow case +// repairDeleteWithPayloads below documents. +func (s *StorageImpl) repairDelete(ctx context.Context, conn *sqlite.Conn, key string) error { + return s.write(ctx, conn, priorityLow, holdPathRepair, resourceFromKey(key), false, func(_ context.Context, conn *sqlite.Conn) error { + return DeleteMetadata(conn, key, nil) + }) +} + +// repairDeleteWithPayloads prunes a superseded SQL payload before metadata. +// Only the missing-file branch calls it, after establishing rv IS NULL and +// no time-series marker. Corrupt-file repairs must preserve recovery bytes. +func (s *StorageImpl) repairDeleteWithPayloads(ctx context.Context, conn *sqlite.Conn, key string) error { + return s.write(ctx, conn, priorityLow, holdPathRepair, resourceFromKey(key), false, func(_ context.Context, conn *sqlite.Conn) error { + if err := DeletePayloads(conn, key); err != nil { + return err + } + return DeleteMetadata(conn, key, nil) + }) +} + // migrateObject runs the external migration tool and unmarshals the output into objPtr. // It is used by get() to migrate objects that need external migration. // The caller must hold the write lock for key before calling this. @@ -923,7 +1257,7 @@ func (s *StorageImpl) migrateObject(ctx context.Context, conn *sqlite.Conn, path migrationCtx, migrationCancel := context.WithTimeout(ctx, 30*time.Second) defer migrationCancel() - cmd := exec.CommandContext(migrationCtx, "/usr/bin/migration", "-file", makePayloadPath(path), "-type", typeName) + cmd := exec.CommandContext(migrationCtx, migrationBinaryPath, "-file", makePayloadPath(path), "-type", typeName) var out bytes.Buffer var stderr bytes.Buffer cmd.Stdout = &out @@ -935,7 +1269,7 @@ func (s *StorageImpl) migrateObject(ctx context.Context, conn *sqlite.Conn, path } logger.L().Ctx(ctx).Error("Get - migration tool failed", helpers.Error(runErr), helpers.String("stderr", stderr.String()), helpers.String("key", key)) // If migration tool fails, treat as corrupted and delete - _ = DeleteMetadata(conn, key, nil) + _ = s.repairDelete(ctx, conn, key) _ = s.appFs.Remove(makePayloadPath(path)) if opts.IgnoreNotFound { return runtime.SetZeroValue(objPtr) @@ -952,7 +1286,7 @@ func (s *StorageImpl) migrateObject(ctx context.Context, conn *sqlite.Conn, path logger.L().Ctx(ctx).Info("Get - external migration successful", helpers.String("key", key)) - if _, saveErr := s.saveObject(conn, key, objPtr, nil, ""); saveErr != nil { + if _, saveErr := s.saveObject(ctx, conn, key, objPtr, nil, "", priorityLow, holdPathMigrate); saveErr != nil { logger.L().Ctx(ctx).Error("Get - failed to rewrite migrated object", helpers.Error(saveErr), helpers.String("key", key)) } else { logger.L().Ctx(ctx).Info("Get - successfully migrated object to modern format", helpers.String("key", key)) @@ -980,11 +1314,11 @@ func (s *StorageImpl) tryDecodePayload(path string, objPtr runtime.Object) (bool return true, nil } -// migrationBinaryPath is the external migration tool invoked by -// execMigrationTool (the hasReadLock/noLock, unlocked-exec path only -- -// migrateObject's own, unchanged exec call for the hasWriteLock path keeps -// its hardcoded path). Package-level var, not const, so tests can point it -// at a fixture script instead of the real /usr/bin/migration binary. +// migrationBinaryPath is the external migration tool invoked on every +// gob-migration path: execMigrationTool (get()'s hasReadLock/noLock states), +// migrateObject (hasWriteLock) and appendGobObjectFromFile (the list readers). +// Package-level var, not const, so tests can point it at a fixture script +// instead of the real /usr/bin/migration binary. var migrationBinaryPath = "/usr/bin/migration" // execMigrationTool runs the external migration binary against path and @@ -1097,7 +1431,7 @@ func (s *StorageImpl) migrateObjectUnlocked(ctx context.Context, conn *sqlite.Co return execErr } logger.L().Ctx(ctx).Error("Get - migration tool failed", helpers.Error(execErr), helpers.String("key", key)) - _ = DeleteMetadata(conn, key, nil) + _ = s.repairDelete(ctx, conn, key) _ = s.appFs.Remove(makePayloadPath(path)) if opts.IgnoreNotFound { return runtime.SetZeroValue(objPtr) @@ -1109,7 +1443,7 @@ func (s *StorageImpl) migrateObjectUnlocked(ctx context.Context, conn *sqlite.Co return unmarshalErr } logger.L().Ctx(ctx).Info("Get - external migration successful", helpers.String("key", key)) - if _, saveErr := s.saveObject(conn, key, objPtr, nil, ""); saveErr != nil { + if _, saveErr := s.saveObject(ctx, conn, key, objPtr, nil, "", priorityLow, holdPathMigrate); saveErr != nil { logger.L().Ctx(ctx).Error("Get - failed to rewrite migrated object", helpers.Error(saveErr), helpers.String("key", key)) } else { logger.L().Ctx(ctx).Info("Get - successfully migrated object to modern format", helpers.String("key", key)) @@ -1339,6 +1673,9 @@ func (s *StorageImpl) fetchListPage(ctx context.Context, conn *sqlite.Conn, key var pageLast string var err error if isFullSpec { + if err := s.refuseForeign("list", key); err != nil { + return listPageResult{}, err + } // get names from SQLite entries, pageLast, err = listMetadataKeys(conn, key, cursor, remaining) } else { @@ -1393,6 +1730,9 @@ func setListContinue(listObj runtime.Object, pageLast string) error { // getListWithSpec is the same as GetList, but it returns the full objects instead of just the metadata. func (s *StorageImpl) getListWithSpec(ctx context.Context, key string, _ storage.ListOptions, listObj runtime.Object) error { + if err := s.refuseForeign("list", key); err != nil { + return err + } ctx, span := otel.Tracer("").Start(ctx, "StorageImpl.getListWithSpec") span.SetAttributes(attribute.String("key", key)) defer span.End() @@ -1498,6 +1838,12 @@ func (s *StorageImpl) GuaranteedUpdate( if singleWriterEnabled { return s.guaranteedUpdateSingleWriter(ctx, key, metaOut, ignoreNotFound, preconditions, tryUpdate, cachedExistingObject, "", priorityHigh) } + if s.gate != nil { + if err := s.refuseForeign("update", key); err != nil { + return err + } + return ErrGateRequiresSingleWriter + } poolCtx, cancel := poolContext() defer cancel() beforePool := time.Now() @@ -1514,6 +1860,9 @@ func (s *StorageImpl) GuaranteedUpdate( func (s *StorageImpl) GuaranteedUpdateWithConn( ctx context.Context, conn *sqlite.Conn, key string, metaOut runtime.Object, ignoreNotFound bool, preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, cachedExistingObject runtime.Object, checksum string) error { + if err := s.refuseForeign("update", key); err != nil { + return err + } ctx, span := otel.Tracer("").Start(ctx, "StorageImpl.GuaranteedUpdate") span.SetAttributes(attribute.String("key", key)) defer span.End() @@ -1680,7 +2029,7 @@ func (s *StorageImpl) GuaranteedUpdateWithConn( } // save to disk and fill into metaOut - metaEvent, err := s.saveObject(conn, key, ret, metaOut, checksum) + metaEvent, err := s.saveObject(ctx, conn, key, ret, metaOut, checksum, priorityHigh, holdPathLegacyCommit) if err != nil { logger.L().Ctx(ctx).Error("GuaranteedUpdate - save object failed", helpers.Error(err), helpers.String("key", key)) return err @@ -1738,6 +2087,9 @@ func (s *StorageImpl) GetByCluster(ctx context.Context, apiVersion, kind string, // appendGobObjectFromFile unmarshalls a Gob file into a runtime.Object and appends it to the underlying list object. func (s *StorageImpl) appendGobObjectFromFile(ctx context.Context, path string, v reflect.Value) error { key := s.keyFromPath(path) + if err := s.refuseForeign("list", key); err != nil { + return err + } lockCtx, lockCancel := context.WithTimeout(ctx, lockTimeout) defer lockCancel() err := s.locks.RLock(lockCtx, key) @@ -1799,7 +2151,7 @@ func (s *StorageImpl) appendGobObjectFromFile(ctx context.Context, path string, migrationCtx, migrationCancel := context.WithTimeout(ctx, 30*time.Second) defer migrationCancel() - cmd := exec.CommandContext(migrationCtx, "/usr/bin/migration", "-file", path, "-type", typeName) + cmd := exec.CommandContext(migrationCtx, migrationBinaryPath, "-file", path, "-type", typeName) var out bytes.Buffer var stderr bytes.Buffer cmd.Stdout = &out @@ -1833,7 +2185,7 @@ func (s *StorageImpl) appendGobObjectFromFile(ctx context.Context, path string, } else { metrics.ObservePoolWait(resourceFromKey(key), metrics.OutcomeAcquired, time.Since(beforePool)) defer s.pool.Put(conn) - if _, saveErr := s.saveObject(conn, key, obj, nil, ""); saveErr != nil { + if _, saveErr := s.saveObject(ctx, conn, key, obj, nil, "", priorityLow, holdPathMigrate); saveErr != nil { logger.L().Ctx(ctx).Error("appendGobObjectFromFile - failed to rewrite migrated object", helpers.Error(saveErr), helpers.String("path", path)) } else { logger.L().Ctx(ctx).Info("appendGobObjectFromFile - successfully migrated object to modern format", helpers.String("path", path)) diff --git a/pkg/registry/file/storage_test.go b/pkg/registry/file/storage_test.go index bee064377..a696c624d 100644 --- a/pkg/registry/file/storage_test.go +++ b/pkg/registry/file/storage_test.go @@ -774,7 +774,7 @@ func BenchmarkWriteFiles(b *testing.B) { metaOut := &v1beta1.SBOMSyft{} conn, _ := s.pool.Take(context.Background()) for i := 0; i < b.N; i++ { - _, _ = s.saveObject(conn, key, obj, metaOut, "") + _, _ = s.saveObject(context.Background(), conn, key, obj, metaOut, "", priorityLow, holdPathLegacyCommit) } s.pool.Put(conn) b.ReportAllocs() @@ -1078,7 +1078,7 @@ func TestStorageImpl_MigrateObjectUnlocked_ConcurrentWriteWins(t *testing.T) { ObjectMeta: v1.ObjectMeta{Name: "toto"}, Spec: v1beta1.SBOMSyftSpec{Metadata: v1beta1.SPDXMeta{Tool: v1beta1.ToolMeta{Name: "concurrent-writer"}}}, } - _, saveErr := s.saveObject(conn, key, newObj, nil, "") + _, saveErr := s.saveObject(context.Background(), conn, key, newObj, nil, "", priorityLow, holdPathLegacyCommit) require.NoError(t, saveErr) pool.Put(conn) s.locks.Unlock(key) @@ -1588,3 +1588,32 @@ func TestStorageImpl_GetList_ReleasesConnectionBetweenPages(t *testing.T) { require.Len(t, list.Items, 1) assert.Equal(t, "sbom-01", list.Items[0].Name) } + +// TestKeyFromPayloadPath_RoundTripsForEveryRootShape: keyFromPayloadPath +// recovers the exact key makePayloadPath(root, key) was written under, +// regardless of root's textual form -- the bug this replaces (slicing a +// Walk()-reported path at len(root)) silently mis-derived the key whenever +// root's length didn't match what filepath.Join actually consumed: a +// trailing slash overcounts by one, root "/" or "." each undercount or +// overcount differently again. filepath.Rel is exact for all of them, which +// this proves by reconstructing path the same way afero.Walk would report +// it (filepath.Join(root, key) + GobExt) for every root shape a real +// cpexport -root flag can be given, then recovering key from it. +func TestKeyFromPayloadPath_RoundTripsForEveryRootShape(t *testing.T) { + keys := []string{ + "/spdx.softwarecomposition.kubescape.io/containerprofile/kube-system/plain-00", + "/spdx.softwarecomposition.kubescape.io/containerprofile/default/app-with-dots.and-dashes", + } + roots := []string{"/data", "/data/", ".", "/", "/data/../data", "data"} + for _, root := range roots { + for _, key := range keys { + t.Run(fmt.Sprintf("root=%q/key=%s", root, key), func(t *testing.T) { + path := makePayloadPath(filepath.Join(root, key)) + got, err := keyFromPayloadPath(root, path) + require.NoError(t, err) + assert.Equal(t, key, got, + "root=%q path=%q: recovered key must match the original, regardless of root's textual form", root, path) + }) + } + } +} diff --git a/pkg/registry/file/testdata/perfab.thresholds.json b/pkg/registry/file/testdata/perfab.thresholds.json new file mode 100644 index 000000000..c871d957b --- /dev/null +++ b/pkg/registry/file/testdata/perfab.thresholds.json @@ -0,0 +1,34 @@ +{ + "_comment": "Tier B pre-registered thresholds (A.13.4 B.5). Relative only; no absolute latency is checked in. ratio: regression when HEAD/BASE breaches threshold_pct in the bad direction AND either the paired t-test or Mann-Whitney U has p < alpha. count: regression when sum(HEAD) > sum(BASE) and significant. points: regression when the mean paired difference exceeds threshold_points and significant. hard: any non-zero value on HEAD in any round while BASE has none in every round, no statistics. info: reported with its statistics, never gates. REST GuaranteedUpdate is gated on p95, not p99: under the pinned production shape about 1% of updates hit acquireLockedConn's 250 ms connection-attempt cliff, so its p99 straddles that cliff and is bimodal round to round (~40 ms vs ~280 ms); the cliff itself is measured by pool-wait-timeouts and ops/s. REST List is gated on p95 for the same reason: the legacy fullSpec page opens one payload file per object and its p99 varies 2-3x round to round (probe CV 51%) while p95 is stable. tick-p50-ms is info (harness v4): once the updaters perform a real mutation, the legacy arm's consolidation pass is bimodal round to round (tick p50 785 ms to 6481 ms in one 12-pair run, 'database is locked' churn under contention) while the ObjectStore arm's stays within ~10% CV; the paired CV of the log-ratio is then a property of the legacy baseline, so gating on it cannot produce a PASS at any N. Consolidation stays gated on tick-p99-ms.", + "alpha": 0.05, + "power": 0.8, + "paired_cv_max_pct": 25, + "probe_cv_max_pct": 50, + "metrics": [ + {"name": "get-p99-ms", "kind": "ratio", "direction": "lower_is_better", "threshold_pct": 20, "headline": true}, + {"name": "create-p99-ms", "kind": "ratio", "direction": "lower_is_better", "threshold_pct": 20, "headline": true}, + {"name": "update-p95-ms", "kind": "ratio", "direction": "lower_is_better", "threshold_pct": 20, "headline": true}, + {"name": "update-p99-ms", "kind": "info", "direction": "lower_is_better"}, + {"name": "list-p95-ms", "kind": "ratio", "direction": "lower_is_better", "threshold_pct": 20, "headline": true}, + {"name": "list-p99-ms", "kind": "info", "direction": "lower_is_better"}, + {"name": "list-p50-ms", "kind": "info", "direction": "lower_is_better"}, + {"name": "list-meta-p95-ms", "kind": "info", "direction": "lower_is_better"}, + {"name": "list-full-p95-ms", "kind": "info", "direction": "lower_is_better"}, + {"name": "write-bytes", "kind": "info", "direction": "lower_is_better"}, + {"name": "tick-p50-ms", "kind": "info", "direction": "lower_is_better"}, + {"name": "tick-p99-ms", "kind": "ratio", "direction": "lower_is_better", "threshold_pct": 30, "headline": true}, + {"name": "tick-total-s", "kind": "info", "direction": "lower_is_better"}, + {"name": "ops-per-s", "kind": "ratio", "direction": "higher_is_better", "threshold_pct": 10, "headline": true}, + {"name": "lock-wait-timeouts", "kind": "count", "direction": "lower_is_better"}, + {"name": "pool-wait-timeouts", "kind": "count", "direction": "lower_is_better"}, + {"name": "over-one-sec", "kind": "count", "direction": "lower_is_better"}, + {"name": "commit-conflict-rate-pct", "kind": "points", "direction": "lower_is_better", "threshold_points": 5}, + {"name": "legacy-create-p99-ms", "kind": "info", "direction": "lower_is_better"}, + {"name": "legacy-update-p99-ms", "kind": "info", "direction": "lower_is_better"}, + {"name": "legacy-delete-p99-ms", "kind": "info", "direction": "lower_is_better"}, + {"name": "cleanup-tick-ms", "kind": "info", "direction": "lower_is_better"}, + {"name": "legacy-over-one-sec", "kind": "count", "direction": "lower_is_better"}, + {"name": "busy-wait-max-ms", "kind": "info", "direction": "lower_is_better"} + ], + "hard": ["err-other", "over-five-sec", "commit-panic", "legacy-err-other", "ungated-writes"] +} diff --git a/pkg/registry/file/testdata/workbudget.golden.json b/pkg/registry/file/testdata/workbudget.golden.json new file mode 100644 index 000000000..0cdca653b --- /dev/null +++ b/pkg/registry/file/testdata/workbudget.golden.json @@ -0,0 +1,203 @@ +{ + "S1_learning_tick": { + "stmt": { + "DeleteMetadata": 3, + "DeletePayloads": 3, + "DeleteTimeSeriesContainerEntries": 3, + "ListTimeSeriesContainers": 1, + "ListTimeSeriesExpired": 1, + "ListTimeSeriesWithData": 1, + "ReadMetadata": 5, + "ReplaceTimeSeriesContainerEntries": 1, + "Save:saveObject": 1, + "Transaction": 1, + "WriteJSON": 1, + "WriteTimeSeriesEntry": 1 + }, + "lock": { + "rlock": 6, + "lock": 4, + "timeout": 0 + }, + "pool_take": 5, + "fs": { + "open": 8, + "rename": 1, + "remove": 3 + }, + "events": { + "Added": 0, + "Modified": 1, + "Deleted": 3 + } + }, + "S2_empty_tick": { + "stmt": { + "ListTimeSeriesExpired": 1, + "ListTimeSeriesWithData": 1 + }, + "lock": { + "rlock": 0, + "lock": 0, + "timeout": 0 + }, + "pool_take": 1, + "fs": { + "open": 0, + "rename": 0, + "remove": 0 + }, + "events": { + "Added": 0, + "Modified": 0, + "Deleted": 0 + } + }, + "S3_frozen_tick": { + "stmt": { + "DeleteMetadata": 2, + "DeletePayloads": 2, + "DeleteTimeSeriesContainerEntries": 2, + "ListTimeSeriesContainers": 1, + "ListTimeSeriesExpired": 1, + "ListTimeSeriesWithData": 1, + "ReadMetadata": 1, + "ReplaceTimeSeriesContainerEntries": 1, + "Transaction": 1 + }, + "lock": { + "rlock": 1, + "lock": 2, + "timeout": 0 + }, + "pool_take": 4, + "fs": { + "open": 1, + "rename": 0, + "remove": 2 + }, + "events": { + "Added": 0, + "Modified": 0, + "Deleted": 2 + } + }, + "S4_divergent_tick": { + "stmt": { + "DeleteMetadata": 2, + "DeletePayloads": 2, + "DeleteTimeSeriesContainerEntries": 2, + "ListTimeSeriesContainers": 1, + "ListTimeSeriesExpired": 1, + "ListTimeSeriesWithData": 1, + "ReadMetadata": 2, + "ReplaceTimeSeriesContainerEntries": 1, + "Save:saveObject": 1, + "Transaction": 1, + "WriteJSON": 1 + }, + "lock": { + "rlock": 1, + "lock": 3, + "timeout": 0 + }, + "pool_take": 4, + "fs": { + "open": 3, + "rename": 1, + "remove": 2 + }, + "events": { + "Added": 0, + "Modified": 1, + "Deleted": 2 + } + }, + "S5_rest_get": { + "stmt": {}, + "lock": { + "rlock": 1, + "lock": 0, + "timeout": 0 + }, + "pool_take": 1, + "fs": { + "open": 1, + "rename": 0, + "remove": 0 + }, + "events": { + "Added": 0, + "Modified": 0, + "Deleted": 0 + } + }, + "S6_rest_create_ts": { + "stmt": { + "ReadMetadata": 2, + "Save:commit": 1, + "WriteJSON": 1, + "WriteTimeSeriesEntry": 1 + }, + "lock": { + "rlock": 0, + "lock": 1, + "timeout": 0 + }, + "pool_take": 3, + "fs": { + "open": 1, + "rename": 1, + "remove": 0 + }, + "events": { + "Added": 1, + "Modified": 0, + "Deleted": 0 + } + }, + "S7_noop_guaranteed_update": { + "stmt": { + "ReadMetadata": 4 + }, + "lock": { + "rlock": 3, + "lock": 0, + "timeout": 0 + }, + "pool_take": 2, + "fs": { + "open": 3, + "rename": 0, + "remove": 0 + }, + "events": { + "Added": 0, + "Modified": 0, + "Deleted": 0 + } + }, + "S8_delete_ts": { + "stmt": { + "DeleteMetadata": 1, + "DeletePayloads": 1, + "DeleteTimeSeriesContainerEntries": 1 + }, + "lock": { + "rlock": 0, + "lock": 1, + "timeout": 0 + }, + "pool_take": 2, + "fs": { + "open": 0, + "rename": 0, + "remove": 1 + }, + "events": { + "Added": 0, + "Modified": 0, + "Deleted": 1 + } + } +} diff --git a/pkg/registry/file/utils.go b/pkg/registry/file/utils.go index 993676ed7..1f8ef9c5c 100644 --- a/pkg/registry/file/utils.go +++ b/pkg/registry/file/utils.go @@ -2,6 +2,8 @@ package file import ( "bytes" + "context" + "encoding/json" "errors" "fmt" "path/filepath" @@ -47,17 +49,47 @@ func NewKubernetesClient() (*kubernetes.Clientset, error) { return kubernetes.NewForConfig(clusterConfig) } -func (h *ResourcesCleanupHandler) deleteMetadata(conn *sqlite.Conn, path string) (runtime.Object, error) { +// deleteMetadata deletes the row (and the time_series rows of a +// containerprofile) behind a reclaimed payload file (W9a of +// write-gate-sharing §3.2): today's autocommit statements on the walk's +// connection with no gate; one gated transaction with the row's JSON decoded +// after release with one. +func (h *ResourcesCleanupHandler) deleteMetadata(ctx context.Context, conn *sqlite.Conn, path string) (runtime.Object, error) { key := payloadPathToKey(path) metaOut := &PartialObjectMetadata{} - err := DeleteMetadata(conn, key, metaOut) + _, _, kind, _, _, _ := K8sPathToKeys(key) + if h.gate == nil { + err := DeleteMetadata(conn, key, metaOut) + if err != nil { + return nil, fmt.Errorf("failed to delete metadata: %w", err) + } + if IsContainerProfileKind(kind) { + if err := DeleteTimeSeriesContainerEntries(conn, key); err != nil { + return nil, fmt.Errorf("failed to delete time series entries: %w", err) + } + } + return metaOut, nil + } + var raw []byte + err := h.write(ctx, conn, holdPathCleanup, resourceFromKey(key), func(_ context.Context, conn *sqlite.Conn) error { + var derr error + raw, derr = deleteMetadataRaw(conn, key) + if derr != nil { + return derr + } + if IsContainerProfileKind(kind) { + if terr := DeleteTimeSeriesContainerEntries(conn, key); terr != nil { + return fmt.Errorf("failed to delete time series entries: %w", terr) + } + } + return nil + }) if err != nil { return nil, fmt.Errorf("failed to delete metadata: %w", err) } - _, _, kind, _, _, _ := K8sPathToKeys(key) - if IsContainerProfileKind(kind) { - if err := DeleteTimeSeriesContainerEntries(conn, key); err != nil { - return nil, fmt.Errorf("failed to delete time series entries: %w", err) + if len(raw) > 0 { + if err := json.Unmarshal(raw, metaOut); err != nil { + return nil, fmt.Errorf("failed to delete metadata: %w", err) } } return metaOut, nil @@ -135,7 +167,7 @@ func payloadPathToKey(path string) string { return path[len(DefaultStorageRoot) : len(path)-len(GobExt)] } -func (h *ResourcesCleanupHandler) readMetadata(conn *sqlite.Conn, payloadFilePath string) (*metav1.ObjectMeta, error) { +func (h *ResourcesCleanupHandler) readMetadata(ctx context.Context, conn *sqlite.Conn, payloadFilePath string) (*metav1.ObjectMeta, error) { key := payloadPathToKey(payloadFilePath) metadataJSON, err := ReadMetadata(conn, key) if err == nil { @@ -153,8 +185,10 @@ func (h *ResourcesCleanupHandler) readMetadata(conn *sqlite.Conn, payloadFilePat h.deleteFunc(h.appFs, payloadFilePath) return nil, fmt.Errorf("failed to read metadata file: %w", err) } - // write to SQLite - err = WriteJSON(conn, key, metadataJSON) + // write to SQLite (W9b: through the gate when there is one) + err = h.write(ctx, conn, holdPathCleanupMigrate, resourceFromKey(key), func(_ context.Context, conn *sqlite.Conn) error { + return WriteJSON(conn, key, metadataJSON) + }) if err != nil { return nil, fmt.Errorf("failed to migrate metadata to SQLite: %w", err) } diff --git a/pkg/registry/file/workbudget.go b/pkg/registry/file/workbudget.go new file mode 100644 index 000000000..e9fbc7507 --- /dev/null +++ b/pkg/registry/file/workbudget.go @@ -0,0 +1,29 @@ +package file + +import "sync/atomic" + +// stmtObserver is the Tier A work-budget hook (TestWorkBudget): every +// sqlitex.Execute call site in this package and every transaction opener +// reports its enclosing function's name through observeStmt. It is nil in +// production, where observeStmt is one atomic load and a nil check and +// allocates nothing. An atomic.Pointer rather than a bare func var so the +// harness can install and clear it while a leaked shard or watch goroutine +// from an earlier test is still executing statements, without a data race. +var stmtObserver atomic.Pointer[func(string)] + +// observeStmt reports one statement (or transaction opener) execution at the +// named site to the installed observer, if any. +func observeStmt(site string) { + if f := stmtObserver.Load(); f != nil { + (*f)(site) + } +} + +// setStmtObserver installs f as the statement observer; nil uninstalls it. +func setStmtObserver(f func(string)) { + if f == nil { + stmtObserver.Store(nil) + return + } + stmtObserver.Store(&f) +} diff --git a/pkg/registry/file/workbudget_test.go b/pkg/registry/file/workbudget_test.go new file mode 100644 index 000000000..2ac5fa9ba --- /dev/null +++ b/pkg/registry/file/workbudget_test.go @@ -0,0 +1,681 @@ +package file + +// Tier A of the storage measurement harness: work budgets. +// +// TestWorkBudget runs eight single-goroutine scenarios over the hot paths of +// this package and counts, per scenario, the SQL statements executed by Go +// call site, the per-key lock acquisitions by mode, the connection-pool takes, +// the payload-file operations and the watch events. The counts are compared +// EXACTLY against testdata/workbudget.golden.json. Counts are machine +// independent, so this runs on every `go test ./...` with no env gate. +// +// A golden change is a review item: the commit that changes it states each +// delta and why, and the reviewer checks the attribution. Regenerate with +// +// go test ./pkg/registry/file -run TestWorkBudget -update +// +// Four invariants hold regardless of the golden (see checkFixedInvariants): +// a no-op writes nothing (S2, S7); a frozen tick writes nothing to the base +// (S3); one REST read is one read lock and one connection (S5); the +// single-writer ROLLBACK recovery never fires on a passing path (all). +// +// Design: .omc/plans/raw-write-bypass-elimination.md, A.13.3. + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/kubescape/storage/pkg/utils" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/apiserver/pkg/storage" + "k8s.io/component-base/metrics/legacyregistry" + "zombiezen.com/go/sqlite" +) + +var updateWorkBudget = flag.Bool("update", false, "rewrite testdata/workbudget.golden.json from this run's counts") + +const ( + workBudgetGoldenPath = "testdata/workbudget.golden.json" + poolWaitMetricName = "storage_pool_wait_duration_seconds" + // budgetSettleWindow is how long the observers stay installed after a + // scenario returns: any observation arriving in it is contamination. + budgetSettleWindow = 50 * time.Millisecond + // budgetEventDrainQuiet / budgetEventDrainMax bound the watch drain. + budgetEventDrainQuiet = 50 * time.Millisecond + budgetEventDrainMax = 2 * time.Second +) + +// workBudgetMu serialises budget scenarios: the two hooks and the metrics +// registry are process-global. +var workBudgetMu sync.Mutex + +// frozenTickInvariant enables fixed invariant 2 (a frozen tick writes nothing +// to the base: no rename, no WriteJSON, no Modified). The property is the +// frozen gate's (X-A, kubescape/storage#399); on a tree without it the tick +// merges late reports into a Completed/Full base and rewrites it, which the +// S3 golden row records. Flip to true when #399 lands. +const frozenTickInvariant = false + +type lockBudget struct { + RLock int `json:"rlock"` + Lock int `json:"lock"` + Timeout int `json:"timeout"` +} + +type fsBudget struct { + Open int `json:"open"` + Rename int `json:"rename"` + Remove int `json:"remove"` +} + +type eventBudget struct { + Added int `json:"Added"` + Modified int `json:"Modified"` + Deleted int `json:"Deleted"` +} + +// workBudget is one scenario's row in the golden. Stmt holds only non-zero +// sites; an absent site is zero. +type workBudget struct { + Stmt map[string]int `json:"stmt"` + Lock lockBudget `json:"lock"` + PoolTake int `json:"pool_take"` + Fs fsBudget `json:"fs"` + Events eventBudget `json:"events"` +} + +// budgetRecorder is the sink behind every hook. It is mutex-guarded, never +// touches testing.T, and once frozen records late arrivals by name instead of +// counting them: a hook may fire from a shard goroutine, or from a goroutine +// leaked by an earlier test that loaded the observer pointer before it was +// cleared, after the scenario (or the test) has returned. +type budgetRecorder struct { + mu sync.Mutex + frozen bool + stmt map[string]int + lock lockBudget + fs fsBudget + late []string +} + +func newBudgetRecorder() *budgetRecorder { + return &budgetRecorder{stmt: map[string]int{}} +} + +func (r *budgetRecorder) hit(site string, bump func()) { + r.mu.Lock() + defer r.mu.Unlock() + if r.frozen { + r.late = append(r.late, site) + return + } + bump() +} + +func (r *budgetRecorder) observeStmt(site string) { + r.hit("stmt:"+site, func() { r.stmt[site]++ }) +} + +func (r *budgetRecorder) observeLock(mode, outcome string) { + r.hit("lock:"+mode+"/"+outcome, func() { + switch { + case outcome == utils.LockOutcomeTimeout: + r.lock.Timeout++ + case mode == utils.LockModeRead: + r.lock.RLock++ + default: + r.lock.Lock++ + } + }) +} + +func (r *budgetRecorder) observeFs(op string) { + r.hit("fs:"+op, func() { + switch op { + case "open": + r.fs.Open++ + case "rename": + r.fs.Rename++ + case "remove": + r.fs.Remove++ + } + }) +} + +// freeze stops counting and returns the counts so far. +func (r *budgetRecorder) freeze() (map[string]int, lockBudget, fsBudget) { + r.mu.Lock() + defer r.mu.Unlock() + r.frozen = true + stmt := make(map[string]int, len(r.stmt)) + for k, v := range r.stmt { + stmt[k] = v + } + return stmt, r.lock, r.fs +} + +func (r *budgetRecorder) lateArrivals() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.late...) +} + +// countingFs counts the payload-file operations StorageImpl issues through +// its afero.Fs. One payload read is one open; one save is one open (the +// staged temp file) plus one rename. +type countingFs struct { + afero.Fs + rec *budgetRecorder +} + +func (c *countingFs) Open(name string) (afero.File, error) { + c.rec.observeFs("open") + return c.Fs.Open(name) +} + +func (c *countingFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + c.rec.observeFs("open") + return c.Fs.OpenFile(name, flag, perm) +} + +func (c *countingFs) Rename(oldname, newname string) error { + c.rec.observeFs("rename") + return c.Fs.Rename(oldname, newname) +} + +func (c *countingFs) Remove(name string) error { + c.rec.observeFs("remove") + return c.Fs.Remove(name) +} + +// poolTakeSamples sums the sample count of storage_pool_wait_duration_seconds +// over every label set. *sqlitemigration.Pool is concrete and cannot be +// hooked, so pool takes are the delta of this histogram around a scenario; +// "observed takes" are the sites that call metrics.ObservePoolWait. +func poolTakeSamples(t *testing.T) int { + t.Helper() + families, err := legacyregistry.DefaultGatherer.Gather() + require.NoError(t, err) + total := 0 + for _, mf := range families { + if mf.GetName() != poolWaitMetricName { + continue + } + for _, m := range mf.GetMetric() { + total += int(m.GetHistogram().GetSampleCount()) + } + } + return total +} + +// budgetEnv is one scenario's freshly built storage plus its fixture keys. +type budgetEnv struct { + t *testing.T + ctx context.Context + s *StorageImpl + processor *ContainerProfileProcessor + rec *budgetRecorder + + tpl softwarecomposition.ContainerProfile + baseNm string + ns string + baseKey string + now time.Time +} + +const zeroReportTimestamp = "0001-01-01 00:00:00 +0000 UTC" + +// newBudgetEnv builds the scenario storage on newLoadStorage's production +// shape (pool 10) with Workers 1 (single goroutine, deterministic), a positive +// DeleteThreshold (so the tick's expired listing executes, as in production; +// fixtures are stamped at run time so nothing is expired) and the counting +// filesystem installed before anything is written. +func newBudgetEnv(t *testing.T, rec *budgetRecorder) *budgetEnv { + t.Helper() + s, processor, pool := newLoadStorage(t, DefaultPoolSize) + t.Cleanup(func() { _ = pool.Close() }) + s.appFs = &countingFs{Fs: s.appFs, rec: rec} + processor.Workers = 1 + processor.DeleteThreshold = 24 * time.Hour + + content, err := os.ReadFile("testdata/p1.json") + require.NoError(t, err) + var tpl softwarecomposition.ContainerProfile + require.NoError(t, json.Unmarshal(content, &tpl)) + baseNm, _ := SplitProfileName(tpl.Name) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + t.Cleanup(cancel) + env := &budgetEnv{ + t: t, + ctx: ctx, + s: s, + processor: processor, + rec: rec, + tpl: tpl, + baseNm: baseNm, + ns: tpl.Namespace, + baseKey: "/spdx.softwarecomposition.kubescape.io/containerprofile/" + tpl.Namespace + "/" + baseNm, + now: time.Now().Round(0), + } + // Warm the CollapseConfiguration cache (10 s TTL) so no scenario pays the + // CR lookup: production steady state, and independent of scenario order. + processor.CollapseSettings() + return env +} + +func (e *budgetEnv) tsKey(suffix string) string { return e.baseKey + "-" + suffix } + +// reportTimestamp stamps report n of the series in the same format +// ListTimeSeriesExpired compares against (time.Time.String, local zone), so +// the lexical expiry check is consistent and nothing seeded here is expired. +func (e *budgetEnv) reportTimestamp(n int) string { + return e.now.Add(time.Duration(n-10) * time.Minute).String() +} + +// ts clones the template as report n of its series under suffix. Reports +// chain (previous = report n-1) so consolidation sees one continuous series. +func (e *budgetEnv) ts(suffix string, n int, status, completion string) *softwarecomposition.ContainerProfile { + p := e.tpl.DeepCopy() + p.Name = e.baseNm + "-" + suffix + p.ResourceVersion = "" + prev := zeroReportTimestamp + if n > 1 { + prev = e.reportTimestamp(n - 1) + } + p.Annotations[helpersv1.ReportTimestampMetadataKey] = e.reportTimestamp(n) + p.Annotations[helpersv1.PreviousReportTimestampMetadataKey] = prev + p.Annotations[helpersv1.StatusMetadataKey] = status + p.Annotations[helpersv1.CompletionMetadataKey] = completion + return p +} + +// restCreate is the node-agent write: StorageImpl.Create of a TS profile. +func (e *budgetEnv) restCreate(p *softwarecomposition.ContainerProfile) { + e.t.Helper() + key := "/spdx.softwarecomposition.kubescape.io/containerprofile/" + p.Namespace + "/" + p.Name + require.NoError(e.t, e.s.Create(e.ctx, key, p, nil, 0)) +} + +// seedTSDirect writes a TS profile and its time_series row without PreSave's +// gate -- the shape a Create that raced ahead of consolidation leaves behind +// (a late row under a Completed/Full base cannot be created through REST). +func (e *budgetEnv) seedTSDirect(p *softwarecomposition.ContainerProfile) { + e.t.Helper() + conn, err := e.s.pool.Take(e.ctx) + require.NoError(e.t, err) + defer e.s.pool.Put(conn) + _, suffix := SplitProfileName(p.Name) + key := e.tsKey(suffix) + _, err = e.s.saveObject(context.Background(), conn, key, p, nil, "", priorityLow, holdPathLegacyCommit) + require.NoError(e.t, err) + require.NoError(e.t, WriteTimeSeriesEntry(conn, ContainerProfileKind, p.Namespace, e.baseNm, + p.Annotations[helpersv1.ReportSeriesIdMetadataKey], suffix, + p.Annotations[helpersv1.ReportTimestampMetadataKey], + p.Annotations[helpersv1.StatusMetadataKey], + p.Annotations[helpersv1.CompletionMetadataKey], + p.Annotations[helpersv1.PreviousReportTimestampMetadataKey], true)) +} + +func (e *budgetEnv) tick() { + e.t.Helper() + require.NoError(e.t, e.processor.ConsolidateTimeSeries(e.ctx)) +} + +// seedLearningBase creates the base profile the common tick operates on: +// report 1 of the series, consolidated once, leaving a Learning base. +func (e *budgetEnv) seedLearningBase() { + e.t.Helper() + e.restCreate(e.ts("r1", 1, helpersv1.Learning, helpersv1.Partial)) + e.tick() +} + +// seedCompletedFullBase consolidates a Completed/Full report 1, leaving a +// Completed/Full base with no time_series rows. +func (e *budgetEnv) seedCompletedFullBase() { + e.t.Helper() + e.restCreate(e.ts("r1", 1, helpersv1.Completed, helpersv1.Full)) + e.tick() +} + +func (e *budgetEnv) getBase() softwarecomposition.ContainerProfile { + e.t.Helper() + var out softwarecomposition.ContainerProfile + require.NoError(e.t, e.s.Get(e.ctx, e.baseKey, storage.GetOptions{}, &out)) + return out +} + +func (e *budgetEnv) withConn(fn func(conn *sqlite.Conn)) { + e.t.Helper() + conn, err := e.s.pool.Take(e.ctx) + require.NoError(e.t, err) + defer e.s.pool.Put(conn) + fn(conn) +} + +// budgetScenario is one golden row: setup runs unobserved, run is measured. +type budgetScenario struct { + name string + setup func(e *budgetEnv) + run func(e *budgetEnv) +} + +func identityUpdate(input runtime.Object, _ storage.ResponseMeta) (runtime.Object, *uint64, error) { + return input, nil, nil +} + +var budgetScenarios = []budgetScenario{ + { + // The common tick and the per-visited-key floor: a Learning base, + // three new reports of its one series, all with data. + name: "S1_learning_tick", + setup: func(e *budgetEnv) { + e.seedLearningBase() + for n := 2; n <= 4; n++ { + e.restCreate(e.ts(fmt.Sprintf("r%d", n), n, helpersv1.Learning, helpersv1.Partial)) + } + }, + run: func(e *budgetEnv) { e.tick() }, + }, + { + // The tick's own floor: nothing pending, so no key is visited. + name: "S2_empty_tick", + setup: func(e *budgetEnv) {}, + run: func(e *budgetEnv) { e.tick() }, + }, + { + // A Completed/Full base with two late reports that carry data. + name: "S3_frozen_tick", + setup: func(e *budgetEnv) { + e.seedCompletedFullBase() + e.seedTSDirect(e.ts("r2", 2, helpersv1.Learning, helpersv1.Partial)) + e.seedTSDirect(e.ts("r3", 3, helpersv1.Learning, helpersv1.Partial)) + }, + run: func(e *budgetEnv) { e.tick() }, + }, + { + // Divergent base: payload Completed/Full at RV n+1, metadata row + // Learning at RV n, plus S3's two late reports. + name: "S4_divergent_tick", + setup: func(e *budgetEnv) { + e.seedLearningBase() + var rowAtN []byte + e.withConn(func(conn *sqlite.Conn) { + var err error + rowAtN, err = ReadMetadata(conn, e.baseKey) + require.NoError(e.t, err) + }) + require.NoError(e.t, e.s.GuaranteedUpdate(e.ctx, e.baseKey, &softwarecomposition.ContainerProfile{}, false, nil, + func(input runtime.Object, _ storage.ResponseMeta) (runtime.Object, *uint64, error) { + p := input.(*softwarecomposition.ContainerProfile) + p.Annotations[helpersv1.StatusMetadataKey] = helpersv1.Completed + p.Annotations[helpersv1.CompletionMetadataKey] = helpersv1.Full + return p, nil, nil + }, nil)) + e.withConn(func(conn *sqlite.Conn) { + require.NoError(e.t, WriteJSON(conn, e.baseKey, rowAtN)) + }) + e.seedTSDirect(e.ts("r2", 2, helpersv1.Learning, helpersv1.Partial)) + e.seedTSDirect(e.ts("r3", 3, helpersv1.Learning, helpersv1.Partial)) + }, + run: func(e *budgetEnv) { e.tick() }, + }, + { + name: "S5_rest_get", + setup: func(e *budgetEnv) { e.seedLearningBase() }, + run: func(e *budgetEnv) { e.getBase() }, + }, + { + name: "S6_rest_create_ts", + setup: func(e *budgetEnv) { e.seedLearningBase() }, + run: func(e *budgetEnv) { + e.restCreate(e.ts("r2", 2, helpersv1.Learning, helpersv1.Partial)) + }, + }, + { + name: "S7_noop_guaranteed_update", + setup: func(e *budgetEnv) { e.seedLearningBase() }, + run: func(e *budgetEnv) { + require.NoError(e.t, e.s.GuaranteedUpdate(e.ctx, e.baseKey, &softwarecomposition.ContainerProfile{}, true, nil, identityUpdate, nil)) + }, + }, + { + // One processed TS profile deleted the way the consolidation pass + // deletes it, under the worker's connection. + name: "S8_delete_ts", + setup: func(e *budgetEnv) { + e.seedLearningBase() + e.restCreate(e.ts("r2", 2, helpersv1.Learning, helpersv1.Partial)) + }, + run: func(e *budgetEnv) { + ctx, cleanup, err := e.processor.ContainerProfileStorage.WithConnection(e.ctx) + require.NoError(e.t, err) + defer cleanup() + require.NoError(e.t, e.processor.deleteProcessedTimeSeries(ctx, []string{e.tsKey("r2")})) + }, + }, +} + +// runBudgetScenario builds a fresh storage, runs setup unobserved, installs +// the observers, runs the scenario, then settles: drains the watch with a +// deadline, keeps the observers installed for budgetSettleWindow, clears +// them, and takes one final locked read. Any observation after the scenario +// returned -- statement, lock, file op or pool take -- fails the test as +// `contaminated: ` rather than surfacing as a golden mismatch. +func runBudgetScenario(t *testing.T, sc budgetScenario) workBudget { + t.Helper() + workBudgetMu.Lock() + defer workBudgetMu.Unlock() + + // Other tests flip this package var with a deferred restore; under false + // S6/S8 take a different path entirely, which must fail by name. + require.True(t, singleWriterEnabled, "%s: singleWriterEnabled must be true", sc.name) + + rec := newBudgetRecorder() + env := newBudgetEnv(t, rec) + sc.setup(env) + + w, err := env.s.Watch(env.ctx, "/", storage.ListOptions{}) + require.NoError(t, err) + defer w.Stop() + + poolBefore := poolTakeSamples(t) + setStmtObserver(rec.observeStmt) + utils.SetLockObserver(rec.observeLock) + // The counting fs was installed at construction so setup's writes go + // through it too; counting starts when the recorder is unfrozen, which it + // is from construction -- so reset what setup accumulated. + rec.mu.Lock() + rec.stmt = map[string]int{} + rec.lock = lockBudget{} + rec.fs = fsBudget{} + rec.mu.Unlock() + + sc.run(env) + + stmt, lock, fs := rec.freeze() + poolAfter := poolTakeSamples(t) + events := drainWatchEvents(w) + + time.Sleep(budgetSettleWindow) + setStmtObserver(nil) + utils.SetLockObserver(nil) + if late := rec.lateArrivals(); len(late) > 0 { + t.Fatalf("%s: contaminated: %s (%d late observations after the scenario returned)", sc.name, late[0], len(late)) + } + if settled := poolTakeSamples(t); settled != poolAfter { + t.Fatalf("%s: contaminated: pool_take (moved %d -> %d after the scenario returned)", sc.name, poolAfter, settled) + } + + for site, n := range stmt { + if n == 0 { + delete(stmt, site) + } + } + return workBudget{ + Stmt: stmt, + Lock: lock, + PoolTake: poolAfter - poolBefore, + Fs: fs, + Events: events, + } +} + +// drainWatchEvents reads the watcher until it has been quiet for +// budgetEventDrainQuiet, or budgetEventDrainMax in total. The dispatcher's +// send is synchronous to the watcher's inCh; the receive from outCh is not. +func drainWatchEvents(w watch.Interface) eventBudget { + var ev eventBudget + deadline := time.NewTimer(budgetEventDrainMax) + defer deadline.Stop() + quiet := time.NewTimer(budgetEventDrainQuiet) + defer quiet.Stop() + for { + select { + case e, ok := <-w.ResultChan(): + if !ok { + return ev + } + switch e.Type { + case watch.Added: + ev.Added++ + case watch.Modified: + ev.Modified++ + case watch.Deleted: + ev.Deleted++ + } + if !quiet.Stop() { + <-quiet.C + } + quiet.Reset(budgetEventDrainQuiet) + case <-quiet.C: + return ev + case <-deadline.C: + return ev + } + } +} + +// checkFixedInvariants are the floors no golden update can lower. +func checkFixedInvariants(t *testing.T, name string, b workBudget) { + t.Helper() + // 4: the dirty-connection ROLLBACK recovery never fires on a passing path. + require.Zero(t, b.Stmt["ROLLBACK"], "%s: invariant 4: ROLLBACK must be 0 in every scenario", name) + switch name { + case "S2_empty_tick", "S7_noop_guaranteed_update": + // 1: a no-op writes nothing. + require.Zero(t, b.Fs.Rename, "%s: invariant 1: a no-op renames nothing", name) + require.Zero(t, b.Stmt["WriteJSON"], "%s: invariant 1: a no-op writes no metadata", name) + require.Equal(t, eventBudget{}, b.Events, "%s: invariant 1: a no-op emits no events", name) + case "S3_frozen_tick": + // 2: a frozen tick writes nothing to the base. + if frozenTickInvariant { + require.Zero(t, b.Fs.Rename, "%s: invariant 2: a frozen tick renames nothing", name) + require.Zero(t, b.Stmt["WriteJSON"], "%s: invariant 2: a frozen tick writes no metadata", name) + require.Zero(t, b.Events.Modified, "%s: invariant 2: a frozen tick modifies nothing", name) + } + case "S5_rest_get": + // 3: one REST read is one read lock and one connection. + require.Equal(t, 1, b.Lock.RLock, "%s: invariant 3: one REST read is one read lock", name) + require.Zero(t, b.Lock.Lock, "%s: invariant 3: a REST read takes no write lock", name) + require.Equal(t, 1, b.PoolTake, "%s: invariant 3: one REST read is one connection", name) + } +} + +func TestWorkBudget(t *testing.T) { + got := make(map[string]workBudget, len(budgetScenarios)) + for _, sc := range budgetScenarios { + got[sc.name] = runBudgetScenario(t, sc) + checkFixedInvariants(t, sc.name, got[sc.name]) + } + + if *updateWorkBudget { + data, err := json.MarshalIndent(got, "", " ") + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Dir(workBudgetGoldenPath), 0o755)) + require.NoError(t, os.WriteFile(workBudgetGoldenPath, append(data, '\n'), 0o644)) + t.Logf("wrote %s", workBudgetGoldenPath) + return + } + + data, err := os.ReadFile(workBudgetGoldenPath) + require.NoError(t, err, "missing golden; run with -update to record it") + var want map[string]workBudget + require.NoError(t, json.Unmarshal(data, &want)) + if diff := diffWorkBudgets(want, got); diff != "" { + t.Fatalf("work budget differs from %s (golden -> got); a golden change is a review item, state each delta and why:\n%s", workBudgetGoldenPath, diff) + } +} + +// diffWorkBudgets lists every field that differs, one line each, in a stable +// order. Empty means equal. +func diffWorkBudgets(want, got map[string]workBudget) string { + var lines []string + names := map[string]struct{}{} + for n := range want { + names[n] = struct{}{} + } + for n := range got { + names[n] = struct{}{} + } + sorted := make([]string, 0, len(names)) + for n := range names { + sorted = append(sorted, n) + } + sort.Strings(sorted) + for _, n := range sorted { + w, wok := want[n] + g, gok := got[n] + if !wok || !gok { + lines = append(lines, fmt.Sprintf("%s: in golden=%v in run=%v", n, wok, gok)) + continue + } + sites := map[string]struct{}{} + for s := range w.Stmt { + sites[s] = struct{}{} + } + for s := range g.Stmt { + sites[s] = struct{}{} + } + siteList := make([]string, 0, len(sites)) + for s := range sites { + siteList = append(siteList, s) + } + sort.Strings(siteList) + for _, s := range siteList { + if w.Stmt[s] != g.Stmt[s] { + lines = append(lines, fmt.Sprintf("%s.stmt.%s: %d -> %d", n, s, w.Stmt[s], g.Stmt[s])) + } + } + cmp := func(field string, a, b int) { + if a != b { + lines = append(lines, fmt.Sprintf("%s.%s: %d -> %d", n, field, a, b)) + } + } + cmp("lock.rlock", w.Lock.RLock, g.Lock.RLock) + cmp("lock.lock", w.Lock.Lock, g.Lock.Lock) + cmp("lock.timeout", w.Lock.Timeout, g.Lock.Timeout) + cmp("pool_take", w.PoolTake, g.PoolTake) + cmp("fs.open", w.Fs.Open, g.Fs.Open) + cmp("fs.rename", w.Fs.Rename, g.Fs.Rename) + cmp("fs.remove", w.Fs.Remove, g.Fs.Remove) + cmp("events.Added", w.Events.Added, g.Events.Added) + cmp("events.Modified", w.Events.Modified, g.Events.Modified) + cmp("events.Deleted", w.Events.Deleted, g.Events.Deleted) + } + return strings.Join(lines, "\n") +} diff --git a/pkg/registry/file/writegate_acg2_on_test.go b/pkg/registry/file/writegate_acg2_on_test.go new file mode 100644 index 000000000..582556ea9 --- /dev/null +++ b/pkg/registry/file/writegate_acg2_on_test.go @@ -0,0 +1,159 @@ +package file + +// AC-G2(on): the flag-on topology (config.ContainerProfileSqliteBackend) — +// the ObjectStore and the legacy StorageImpl over one pool, the legacy +// instance carrying the kind-ownership guard, one write gate. The lock +// holder is the gate itself (CR-3). Repair readers must queue behind an +// explicitly held gate and remain pending until release. Non-repair readers +// must complete before release. The armed AC-G1 check rejects ungated writes. + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/armosec/armoapi-go/armotypes" + helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/k8s-interface/names" + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + "zombiezen.com/go/sqlite" +) + +// newACG2OnEnv builds the flag-on environment: pool size ≥ 2 (R-9: the +// reader holds its pool connection while queued on the gate; the gate's +// dedicated connection is the other), the legacy instance with the guard, +// the ObjectStore, the cleanup handler, the fixture handle, and AC-G1 armed. +func newACG2OnEnv(t *testing.T) *acg2Env { + t.Helper() + dbPath := filepath.Join(t.TempDir(), "acg2.sq3") + rec := &tableRecorder{} + pool := NewPoolWithOptions(dbPath, PoolOptions{Size: 4, BusyTimeout: acg2BusyTimeout, DisableAutoCheckpoint: true, Authorizer: rec.authorizer}) + armUngatedWriteCheck(t, pool) + e := newACG2Base(t, pool, dbPath) + t.Cleanup(e.closePool) + e.rec = rec + e.legacy.SetForeignKinds(IsContainerProfileKind) + gate, err := newWriteGate(e.ctx, pool) + require.NoError(t, err) + e.gate = gate + e.legacy.SetWriteGate(gate) + e.cleanup.SetWriteGate(gate) + store, err := NewObjectStore(pool, dbPath, e.legacy.watchDispatcher, e.legacy.scheme, e.processor, e.legacy, gate, ObjectStoreOptions{CheckpointInterval: time.Hour}) + require.NoError(t, err) + e.store = store + e.closeStore = func() { + require.NoError(t, store.Close()) + require.NoError(t, gate.Close()) + } + e.hold = func() func() { + held := make(chan struct{}) + release := make(chan struct{}) + done := make(chan error, 1) + go func() { + done <- e.gate.run(e.ctx, priorityHigh, "acg2-holder", "test", func(context.Context, *sqlite.Conn) error { + close(held) + select { + case <-release: + return nil + case <-e.ctx.Done(): + return e.ctx.Err() + } + }) + }() + select { + case <-held: + case <-time.After(acg2Deadline): + close(release) + t.Fatal("gate holder did not acquire the gate") + } + return sync.OnceFunc(func() { + close(release) + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(acg2Deadline): + t.Fatal("gate holder did not finish") + } + }) + } + + return e +} + +var acg2On = acg2Topology{ + name: "on", + on: true, + build: newACG2OnEnv, +} + +// acg2BaseProfile is testdata/p1.json as a base (consolidated) profile: the +// object PreSave runs the SBOM lookup for. +func acg2BaseProfile(t *testing.T) *softwarecomposition.ContainerProfile { + t.Helper() + content, err := os.ReadFile("testdata/p1.json") + require.NoError(t, err) + var p softwarecomposition.ContainerProfile + require.NoError(t, json.Unmarshal(content, &p)) + p.Name, _ = SplitProfileName(p.Name) + delete(p.Annotations, helpersv1.ReportSeriesIdMetadataKey) + return &p +} + +// acg2SbomKeyOf derives the sbomsyft key PreSave reads for p, exactly as +// ContainerProfileProcessor.PreSave does. +func acg2SbomKeyOf(e *acg2Env, p *softwarecomposition.ContainerProfile) string { + e.t.Helper() + slug, err := names.ImageInfoToSlug(p.Spec.ImageTag, p.Spec.ImageID) + require.NoError(e.t, err) + id := armotypes.ProfileIdentifier{ + ProfileScope: armotypes.ProfileScope{ + HostType: e.processor.HostType, + Cluster: p.Annotations[helpersv1.ClusterMetadataKey], + Namespace: e.processor.DefaultNamespace, + CloudAccountIdentifier: p.Annotations[helpersv1.CloudAccountIdentifierMetadataKey], + Region: p.Annotations[helpersv1.RegionMetadataKey], + HostID: p.Annotations[helpersv1.HostIDMetadataKey], + }, + Name: slug, + } + return BuildContainerProfileKey(id, "sbomsyft") +} + +// acg2OnReaders are the flag-on-only entry points: PreSave for a base CP, +// the path bug 2 sat on (PreSave → GetSbom → legacy GetWithConn → get()). +var acg2OnReaders = []acg2Reader{ + {name: "PreSave(base-CP)", class: acg2GetReader, onOnly: true, + key: func(e *acg2Env, cell string) string { + p := acg2BaseProfile(e.t) + // Include the state marker in the image-derived SBOM path so the + // migration fixture distinguishes tool failure from success. + p.Spec.ImageTag += "-" + cell + e.preSaveProfile = p + return acg2SbomKeyOf(e, p) + }, + obj: newSBOM, + prepare: func(e *acg2Env, key string) func() error { + p := e.preSaveProfile + return func() error { + ctx, cleanup, err := e.processor.ContainerProfileStorage.WithConnection(e.ctx) + if err != nil { + return err + } + defer cleanup() + return e.processor.PreSave(ctx, p) + } + }}, +} + +// TestACG2_FlagOn is AC-G2(on). +func TestACG2_FlagOn(t *testing.T) { + runACG2Matrix(t, acg2On, append(append([]acg2Reader(nil), acg2Readers...), acg2OnReaders...)) +} + +var _ runtime.Object diff --git a/pkg/registry/file/writegate_acg2_test.go b/pkg/registry/file/writegate_acg2_test.go new file mode 100644 index 000000000..9abbd43b6 --- /dev/null +++ b/pkg/registry/file/writegate_acg2_test.go @@ -0,0 +1,549 @@ +package file + +// AC-G2 of .omc/plans/write-gate-sharing.md: every read entry point returns +// promptly while SQLite's write lock is held — the generalisation of the two +// tests #401 added (TestGet_AbsentKeyDoesNotWaitOnWriter and the collapse +// provider's). A table over (read entry point) × (key state), run once per +// topology: +// +// - AC-G2(off) — no gate; a pool connection holds BEGIN IMMEDIATE until +// the reader returns. Repair cells must attempt a write without changing +// metadata; other synchronous readers must not attempt writes. +// - AC-G2(on) — see writegate_acg2_on_test.go. +// +// Key states cover both found bugs' statement on every door it has: absent +// (from #401), present, orphan (row, no file: W4), corrupt gob (row, +// truncated file: W5), wrong-type with a failing migration tool (W6a/W7a, no +// write on the list readers), wrong-type with a succeeding tool (the W6b/ +// W7b/W8 rewrites); and for the cleanup tick, referenced (pure read), +// unreferenced (W9a) and file-without-row (W9b). + +import ( + "bytes" + "context" + "encoding/gob" + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + mapset "github.com/deckarep/golang-set/v2" + "github.com/goradd/maps" + helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/kubescape/storage/pkg/apis/softwarecomposition/install" + "github.com/kubescape/storage/pkg/config" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/storage" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitemigration" + "zombiezen.com/go/sqlite/sqlitex" +) + +const ( + // acg2BusyTimeout lets flag-off repairs finish while the writer stays held. + acg2BusyTimeout = time.Second + // acg2Prompt remains the bound used by the independent gate-refusal test. + acg2Prompt = 500 * time.Millisecond + // This is a deadlock guard, not a read-latency acceptance threshold. + acg2Deadline = 30 * time.Second + + acg2Kind = "sbomsyft" + acg2Group = "spdx.softwarecomposition.kubescape.io" + acg2DefaultNS = "kubescape" // the SBOM namespace (ContainerProfileProcessor.DefaultNamespace) + acg2LiveImage = "sha256:live" + acg2DeadImage = "sha256:dead" +) + +// acg2Topology is what differs between AC-G2(off) and AC-G2(on). +type acg2Topology struct { + name string + on bool + // build constructs a fresh environment for one cell. + build func(t *testing.T) *acg2Env +} + +type acg2Env struct { + t *testing.T + ctx context.Context + dbPath string + fs afero.Fs + pool *sqlitemigration.Pool + legacy *StorageImpl + fixture *sqlite.Conn + processor *ContainerProfileProcessor + preSaveProfile *softwarecomposition.ContainerProfile + cleanup *ResourcesCleanupHandler + fetcher *acg2Fetcher + // hold acquires SQLite's write lock the topology's way and returns the + // release; flag-on repairs are released after they queue. + hold func() (release func()) + // closeStore runs before pool.Close (nil under flag-off). + closeStore func() + // The statement recorder observes both topologies; store and gate are flag-on only. + store *ObjectStore + gate *writeGate + rec *tableRecorder +} + +// acg2Fetcher is the cleanup tick's ResourcesFetcher: one namespace that +// lists the live image ids, plus the default namespace the SBOMs live in. +type acg2Fetcher struct { + live mapset.Set[string] +} + +func (f *acg2Fetcher) ListNamespaces(*sqlite.Conn) ([]string, error) { + return []string{"other", acg2DefaultNS}, nil +} + +func (f *acg2Fetcher) FetchResources(string) (ResourceMaps, error) { + return ResourceMaps{ + RunningContainerImageIds: f.live, + RunningInstanceIds: mapset.NewSet[string](), + RunningTemplateHash: mapset.NewSet[string](), + RunningWlidsToContainerNames: new(maps.SafeMap[string, mapset.Set[string]]), + }, nil +} + +// newACG2Base builds the parts both topologies share: a pool with +// acg2BusyTimeout on every connection, the legacy StorageImpl over an +// in-memory filesystem, the ContainerProfileProcessor (its storage is set by +// the topology), the cleanup handler and the fixture handle. +func newACG2Base(t *testing.T, pool *sqlitemigration.Pool, dbPath string) *acg2Env { + t.Helper() + sch := runtime.NewScheme() + install.Install(sch) + wd := NewWatchDispatcher() + fs := afero.NewMemMapFs() + legacy := NewStorageImpl(fs, DefaultStorageRoot, pool, wd, sch).(*StorageImpl) + processor := NewContainerProfileProcessor(acg2ProcessorConfig(), nil) + processor.Interval = 0 + processor.Workers = 1 + fetcher := &acg2Fetcher{live: mapset.NewSet(acg2LiveImage)} + cleanup := NewResourcesCleanupHandler(fs, DefaultStorageRoot, pool, wd, 0, acg2DefaultNS, fetcher, false) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + t.Cleanup(cancel) + return &acg2Env{ + t: t, ctx: ctx, dbPath: dbPath, fs: fs, pool: pool, legacy: legacy, + fixture: openFixtureConn(t, pool, dbPath, acg2BusyTimeout), + processor: processor, cleanup: cleanup, fetcher: fetcher, + } +} + +// closePool is the env's last cleanup: the store first (K-5), then the pool, +// which must return promptly once every connection is back. +func (e *acg2Env) closePool() { + if e.closeStore != nil { + e.closeStore() + } + done := make(chan error, 1) + go func() { done <- e.pool.Close() }() + select { + case err := <-done: + require.NoError(e.t, err) + case <-time.After(30 * time.Second): + e.t.Errorf("pool.Close did not return: a connection was never returned") + } +} + +// acg2Off is AC-G2(off): today's production topology, no gate. +var acg2Off = acg2Topology{ + name: "off", + build: func(t *testing.T) *acg2Env { + dbPath := filepath.Join(t.TempDir(), "acg2.sq3") + rec := &tableRecorder{} + pool := NewPoolWithOptions(dbPath, PoolOptions{Size: 4, BusyTimeout: acg2BusyTimeout, Authorizer: rec.authorizer}) + e := newACG2Base(t, pool, dbPath) + e.rec = rec + t.Cleanup(e.closePool) + e.processor.SetStorage(NewContainerProfileStorageImpl(e.legacy, pool)) + // The holder: a pool connection in an open BEGIN IMMEDIATE, the state + // a legacy writer is in for the length of its transaction. + e.hold = func() func() { + conn, err := pool.Take(e.ctx) + require.NoError(t, err) + conn.SetInterrupt(nil) + endFn, err := sqlitex.ImmediateTransaction(conn) + require.NoError(t, err) + return sync.OnceFunc(func() { + var txErr error + endFn(&txErr) + require.NoError(t, txErr) + pool.Put(conn) + }) + } + return e + }, +} + +// ---- key states ---- + +func (e *acg2Env) payloadPath(key string) string { + return makePayloadPath(filepath.Join(DefaultStorageRoot, key)) +} + +// seedRow writes the metadata row exactly as writeMetadata does, through the +// fixture handle. +func (e *acg2Env) seedRow(key string, obj runtime.Object) { + e.t.Helper() + raw, err := json.Marshal(extractFields(obj, []string{"ObjectMeta", "SchemaVersion"})) + require.NoError(e.t, err) + require.NoError(e.t, WriteJSON(e.fixture, key, raw)) +} + +func gobBytes(t *testing.T, obj runtime.Object) []byte { + t.Helper() + var buf bytes.Buffer + require.NoError(t, gob.NewEncoder(&buf).Encode(obj)) + return buf.Bytes() +} + +func (e *acg2Env) seedPayload(key string, payload []byte) { + e.t.Helper() + require.NoError(e.t, afero.WriteFile(e.fs, e.payloadPath(key), payload, 0644)) +} + +func (e *acg2Env) seedPresent(key string, obj runtime.Object) { + e.seedRow(key, obj) + e.seedPayload(key, gobBytes(e.t, obj)) +} + +type acg2KeyState struct { + name string + seed func(e *acg2Env, key string, obj runtime.Object) + // cleanupOnly states exist for the cleanup tick's referencedness axis. + cleanupOnly bool +} + +var acg2States = []acg2KeyState{ + {name: "absent", seed: func(*acg2Env, string, runtime.Object) {}}, + {name: "present", seed: func(e *acg2Env, key string, obj runtime.Object) { e.seedPresent(key, obj) }}, + {name: "orphan", seed: func(e *acg2Env, key string, obj runtime.Object) { + e.seedPresent(key, obj) + require.NoError(e.t, e.fs.Remove(e.payloadPath(key))) + }}, + {name: "corrupt", seed: func(e *acg2Env, key string, obj runtime.Object) { + // An empty payload (created, never written: a crash before the first + // O_DIRECT block) decodes to io.EOF, the branch get() treats as + // corrupt (W5). A partially written gob fails with gob's own "extra + // data"/"type" errors, which get() returns without repairing. + e.seedRow(key, obj) + e.seedPayload(key, nil) + }}, + {name: "wrongtype-toolfails", seed: func(e *acg2Env, key string, obj runtime.Object) { + e.seedRow(key, obj) + e.seedPayload(key, gobPayloadNeedingMigration(e.t)) + }}, + {name: "wrongtype-toolsucceeds", seed: func(e *acg2Env, key string, obj runtime.Object) { + e.seedRow(key, obj) + e.seedPayload(key, gobPayloadNeedingMigration(e.t)) + }}, + {name: "present-referenced", cleanupOnly: true, seed: func(e *acg2Env, key string, obj runtime.Object) { + obj.(metav1.Object).SetAnnotations(map[string]string{helpersv1.ImageIDMetadataKey: acg2LiveImage}) + e.seedPresent(key, obj) + }}, + {name: "present-unreferenced", cleanupOnly: true, seed: func(e *acg2Env, key string, obj runtime.Object) { + obj.(metav1.Object).SetAnnotations(map[string]string{helpersv1.ImageIDMetadataKey: acg2DeadImage}) + e.seedPresent(key, obj) + }}, + {name: "file-without-row", cleanupOnly: true, seed: func(e *acg2Env, key string, obj runtime.Object) { + obj.(metav1.Object).SetAnnotations(map[string]string{helpersv1.ImageIDMetadataKey: acg2LiveImage}) + e.seedPayload(key, gobBytes(e.t, obj)) + raw, err := json.Marshal(extractFields(obj, []string{"ObjectMeta", "SchemaVersion"})) + require.NoError(e.t, err) + sidecar := strings.TrimSuffix(e.payloadPath(key), GobExt) + MetadataExt + require.NoError(e.t, afero.WriteFile(e.fs, sidecar, raw, 0644)) + }}, +} + +// installACG2MigrationTool points migrationBinaryPath at one script for the +// whole matrix: it fails for a payload whose path names the tool-fails state +// and prints a valid object otherwise, so cells need no per-cell package +// state and can run in parallel. +func installACG2MigrationTool(t *testing.T) { + t.Helper() + scriptPath := filepath.Join(t.TempDir(), "acg2-migration.sh") + script := "#!/bin/sh\ncase \"$2\" in *toolfails*) echo 'acg2: tool fails' >&2; exit 1;; esac\n" + + "printf '%s' '{\"metadata\":{\"name\":\"migrated\",\"namespace\":\"" + acg2DefaultNS + "\"}}'\n" + require.NoError(t, os.WriteFile(scriptPath, []byte(script), 0755)) + old := migrationBinaryPath + migrationBinaryPath = scriptPath + t.Cleanup(func() { migrationBinaryPath = old }) +} + +// ---- read entry points ---- + +type acg2ReaderClass int + +const ( + // acg2GetReader reaches get(): repairs orphan/corrupt/wrong-type keys. + acg2GetReader acg2ReaderClass = iota + // acg2ListReader walks payload files (appendGobObjectFromFile): the only + // write is the W8 rewrite after a succeeding migration tool. + acg2ListReader + // acg2MetaReader reads the metadata row only: never writes. + acg2MetaReader + // acg2CleanupReader is one cleanup tick: writes on the unreferenced and + // file-without-row states. + acg2CleanupReader +) + +type acg2Reader struct { + name string + class acg2ReaderClass + onOnly bool + // key returns the key whose state the cell seeds; obj a fresh object of + // its kind. + key func(e *acg2Env, cell string) string + obj func() runtime.Object + // prepare, when set, runs before the key is seeded and before the lock is + // held, and returns the measured read (for readers whose construction + // itself reads, like the collapse provider's prime). + prepare func(e *acg2Env, key string) func() error + run func(e *acg2Env, key string) error +} + +func (r acg2Reader) repairs(st acg2KeyState) bool { + switch r.class { + case acg2GetReader: + return st.name == "orphan" || st.name == "corrupt" || strings.HasPrefix(st.name, "wrongtype") + case acg2ListReader: + return st.name == "wrongtype-toolsucceeds" + case acg2CleanupReader: + return st.name == "present-unreferenced" || st.name == "file-without-row" + } + return false +} + +func sbomKey(e *acg2Env, cell string) string { + return K8sKeysToPath("", acg2Group, acg2Kind, "", acg2DefaultNS, cell) +} + +func newSBOM() runtime.Object { + return &softwarecomposition.SBOMSyft{ObjectMeta: metav1.ObjectMeta{Namespace: acg2DefaultNS}} +} + +func sbomPrefix() string { return "/" + acg2Group + "/" + acg2Kind + "/" + acg2DefaultNS } + +// acg2Readers are the entry points common to both topologies; the flag-on +// file appends its own. +var acg2Readers = []acg2Reader{ + {name: "Get", class: acg2GetReader, key: sbomKey, obj: newSBOM, run: func(e *acg2Env, key string) error { + return e.legacy.Get(e.ctx, key, storage.GetOptions{}, &softwarecomposition.SBOMSyft{}) + }}, + {name: "Get(metadata)", class: acg2MetaReader, key: sbomKey, obj: newSBOM, run: func(e *acg2Env, key string) error { + return e.legacy.Get(e.ctx, key, storage.GetOptions{ResourceVersion: softwarecomposition.ResourceVersionMetadata}, &softwarecomposition.SBOMSyft{}) + }}, + {name: "GetWithConn", class: acg2GetReader, key: sbomKey, obj: newSBOM, run: func(e *acg2Env, key string) error { + conn, err := e.pool.Take(e.ctx) + if err != nil { + return err + } + defer e.pool.Put(conn) + return e.legacy.GetWithConn(e.ctx, conn, key, storage.GetOptions{}, &softwarecomposition.SBOMSyft{}) + }}, + {name: "get(noLock)", class: acg2GetReader, key: sbomKey, obj: newSBOM, run: func(e *acg2Env, key string) error { + conn, err := e.pool.Take(e.ctx) + if err != nil { + return err + } + defer e.pool.Put(conn) + return e.legacy.get(e.ctx, conn, key, storage.GetOptions{}, &softwarecomposition.SBOMSyft{}, noLock) + }}, + {name: "GetList(metadata)", class: acg2MetaReader, key: sbomKey, obj: newSBOM, run: func(e *acg2Env, key string) error { + return e.legacy.GetList(e.ctx, sbomPrefix(), storage.ListOptions{ResourceVersion: softwarecomposition.ResourceVersionMetadata, Recursive: true}, &softwarecomposition.SBOMSyftList{}) + }}, + {name: "GetList(fullSpec)", class: acg2GetReader, key: sbomKey, obj: newSBOM, run: func(e *acg2Env, key string) error { + return e.legacy.GetList(e.ctx, sbomPrefix(), storage.ListOptions{ResourceVersion: softwarecomposition.ResourceVersionFullSpec, Recursive: true}, &softwarecomposition.SBOMSyftList{}) + }}, + {name: "GetByNamespace", class: acg2ListReader, key: sbomKey, obj: newSBOM, run: func(e *acg2Env, key string) error { + return e.legacy.GetByNamespace(e.ctx, acg2Group, acg2Kind, acg2DefaultNS, &softwarecomposition.SBOMSyftList{}) + }}, + {name: "GetByCluster", class: acg2ListReader, key: sbomKey, obj: newSBOM, run: func(e *acg2Env, key string) error { + return e.legacy.GetByCluster(e.ctx, acg2Group, acg2Kind, &softwarecomposition.SBOMSyftList{}) + }}, + {name: "Count", class: acg2MetaReader, key: sbomKey, obj: newSBOM, run: func(e *acg2Env, key string) error { + _, err := e.legacy.Count(sbomPrefix()) + return err + }}, + {name: "Watch", class: acg2MetaReader, key: sbomKey, obj: newSBOM, run: func(e *acg2Env, key string) error { + w, err := e.legacy.Watch(e.ctx, key, storage.ListOptions{}) + if err == nil { + w.Stop() + } + return err + }}, + {name: "GetSbom", class: acg2GetReader, key: sbomKey, obj: newSBOM, run: func(e *acg2Env, key string) error { + ctx, cleanup, err := e.processor.ContainerProfileStorage.WithConnection(e.ctx) + if err != nil { + return err + } + defer cleanup() + _, err = e.processor.ContainerProfileStorage.GetSbom(ctx, key) + return err + }}, + {name: "collapse-provider", class: acg2MetaReader, + key: func(*acg2Env, string) string { return collapseConfigurationKey(DefaultCollapseConfigurationName) }, + obj: func() runtime.Object { + return &softwarecomposition.CollapseConfiguration{ObjectMeta: metav1.ObjectMeta{Name: DefaultCollapseConfigurationName}} + }, + prepare: func(e *acg2Env, key string) func() error { + // The provider primes synchronously at construction (on the absent + // key, as in #401's test); the key state is seeded after that, so it + // is the background refresh that meets it. The refresh is off the + // caller's goroutine (#401): the closure is prompt in every key + // state, and its Get is what AC-G1 watches. Both Gets (prime + + // refresh) are joined after the measurement, before the env closes. + cs := &countingGetStorage{Interface: e.legacy} + provider := NewCRDCollapseSettingsProvider(cs) + e.t.Cleanup(func() { joinRefresh(e.t, cs) }) + return func() error { provider(); return nil } + }}, + {name: "cleanup-tick", class: acg2CleanupReader, key: sbomKey, obj: newSBOM, run: func(e *acg2Env, key string) error { + return e.cleanup.CleanupTask(e.ctx, map[string][]TypeCleanupHandlerFunc{acg2Kind: {deleteByImageId}}) + }}, +} + +func acg2ReaderStates(r acg2Reader) []acg2KeyState { + var out []acg2KeyState + for _, st := range acg2States { + if st.cleanupOnly == (r.class == acg2CleanupReader) { + out = append(out, st) + } + } + return out +} + +// runACG2Matrix runs every (reader × key state) cell of topo, each in a fresh +// environment, in parallel. Package state the cells share (the migration +// tool, the collapse TTL) is set once here and restored after the last cell. +func runACG2Matrix(t *testing.T, topo acg2Topology, readers []acg2Reader) { + t.Helper() + installACG2MigrationTool(t) + oldTTL := collapseSettingsTTL + collapseSettingsTTL = time.Nanosecond + t.Cleanup(func() { collapseSettingsTTL = oldTTL }) + + for _, r := range readers { + if r.onOnly && !topo.on { + continue + } + for _, st := range acg2ReaderStates(r) { + r, st := r, st + t.Run(r.name+"/"+st.name, func(t *testing.T) { + t.Parallel() + e := topo.build(t) + cell := strings.NewReplacer("(", "-", ")", "", "/", "-").Replace(r.name + "-" + st.name) + key := r.key(e, cell) + obj := r.obj() + if m, ok := obj.(metav1.Object); ok && m.GetName() == "" { + _, _, _, _, _, name := K8sPathToKeys(key) + m.SetName(name) + } + run := func() error { return r.run(e, key) } + if r.prepare != nil { + run = r.prepare(e, key) + } + st.seed(e, key, obj) + + repair := r.repairs(st) + before, beforeErr := ReadMetadata(e.fixture, key) + if beforeErr != nil { + require.ErrorIs(t, beforeErr, ErrMetadataNotFound) + } + release := e.hold() + // Release before registered cleanups even when an assertion fails. + defer release() + mark := e.rec.mark() + start := time.Now() + done := make(chan error, 1) + stopped := make(chan struct{}) + go func() { + defer close(stopped) + done <- run() + }() + defer func() { + release() + select { + case <-stopped: + case <-time.After(acg2Deadline): + t.Error("reader did not stop after release") + } + }() + if topo.on && repair { + require.Eventually(t, func() bool { + high, low := e.gate.queued() + if high+low > 0 { + return true + } + select { + case <-stopped: + return true + default: + return false + } + }, acg2Deadline, time.Millisecond, "repair must queue behind the held gate") + select { + case err := <-done: + t.Fatalf("repair returned before the gate was released: %v", err) + default: + } + release() + } + var err error + select { + case err = <-done: + case <-time.After(acg2Deadline): + t.Fatal("reader did not complete") + } + elapsed := time.Since(start) + // The collapse provider returns cached settings and starts a separate + // refresh, which may repair. Its contract here is caller completion + // while held; prepare registers a cleanup to join that refresh. + if r.name != "collapse-provider" { + wrote := false + for _, action := range e.rec.since(mark) { + wrote = wrote || isWriteOp(action.op) + } + assert.Equal(t, repair, wrote, "repair cells must attempt writes; pure reads must not") + } + if topo.on && repair && (st.name == "orphan" || st.name == "corrupt" || st.name == "wrongtype-toolfails" || st.name == "present-unreferenced") { + _, afterErr := ReadMetadata(e.fixture, key) + assert.ErrorIs(t, afterErr, ErrMetadataNotFound, "queued repair must delete the metadata after release") + } + if !topo.on && repair { + after, afterErr := ReadMetadata(e.fixture, key) + assert.Equal(t, before, after, "repair cannot change metadata while SQLite's writer is held") + assert.Equal(t, beforeErr, afterErr) + } + release() + + kind := "read" + if repair { + kind = "repair" + } + t.Logf("AC-G2(%s) %s × %s: %s cell, %s (err=%v)", topo.name, r.name, st.name, kind, elapsed, err) + if st.name == "present" && r.class != acg2MetaReader { + assert.NoError(t, err, "a present key must read") + } + }) + } + } +} + +// TestACG2_FlagOff is AC-G2(off): the legacy topology's residual, pinned. +func TestACG2_FlagOff(t *testing.T) { + runACG2Matrix(t, acg2Off, acg2Readers) +} + +// acg2ProcessorConfig is the processor config both topologies use. +func acg2ProcessorConfig() config.Config { + return config.Config{DefaultNamespace: acg2DefaultNS, MaxContainerProfileSize: 40000} +} diff --git a/pkg/registry/file/writegate_fixture_test.go b/pkg/registry/file/writegate_fixture_test.go new file mode 100644 index 000000000..e0f5db7cf --- /dev/null +++ b/pkg/registry/file/writegate_fixture_test.go @@ -0,0 +1,38 @@ +package file + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitemigration" + "zombiezen.com/go/sqlite/sqlitex" +) + +// openFixtureConn opens a connection to pool's database file that is NOT a +// pool connection: the one handle tests may seed state through. A fixture +// write on a pool connection would prepare (and cache) a write statement +// there, and the code under test re-executing the same statement on that +// connection would then never re-invoke the authorizer the write-gate +// invariant (AC-G1) is built on — so fixtures never touch the pool. +// +// The pool's schema migration runs on the first Take; the handle is opened +// only after that, or fixture statements race the migration and fail on a +// missing table. Autocheckpoint is off on the handle so a fixture COMMIT does +// not checkpoint under the tests that measure the background checkpointer. +func openFixtureConn(t *testing.T, pool *sqlitemigration.Pool, dbPath string, busyTimeout time.Duration) *sqlite.Conn { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + c, err := pool.Take(ctx) + require.NoError(t, err, "pool did not become ready") + pool.Put(c) + conn, err := sqlite.OpenConn(dbPath, sqlite.OpenReadWrite|sqlite.OpenWAL) + require.NoError(t, err) + conn.SetBusyTimeout(busyTimeout) + require.NoError(t, sqlitex.ExecuteTransient(conn, `PRAGMA wal_autocheckpoint=0`, nil)) + t.Cleanup(func() { _ = conn.Close() }) + return conn +} diff --git a/pkg/registry/file/writegate_registry.go b/pkg/registry/file/writegate_registry.go new file mode 100644 index 000000000..a8d5cb66d --- /dev/null +++ b/pkg/registry/file/writegate_registry.go @@ -0,0 +1,114 @@ +package file + +// One write gate per pool, and the instrument that proves it is the only +// writer (design: .omc/plans/write-gate-sharing.md §3.6 R-7, §7 AC-G1). +// +// SQLite has one write lock per database. Every write that goes through the +// gate is queued FIFO in Go; every write that does not acquires the same lock +// through SQLite's busy handler, which polls every 1…100 ms, is not +// ctx-bounded, and loses systematically against a gate that commits +// continuously — a stall of the whole busy timeout (60 s in production) that +// the gate's own instruments cannot see. So: +// +// - a pool may carry at most one live gate (two gates on two dedicated +// connections would each gate their own writes while busy-waiting +// against each other — the class, visible from neither side); +// - every INSERT/UPDATE/DELETE prepared on a connection the pool's gate has +// never owned is counted (storage_sqlite_ungated_write_total, the +// production canary) and handed to the test observer that turns it into +// a failure with the stack of the site that prepared it. +// +// *sqlitemigration.Pool is a third-party type with nothing to register on, +// so the registry is a package-level map keyed by the pool pointer. + +import ( + "errors" + "strings" + "sync" + "sync/atomic" + + "github.com/kubescape/storage/pkg/metrics" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitemigration" +) + +// errGateExists is returned by newWriteGate for a pool that already has a +// live gate. +var errGateExists = errors.New("write gate: the pool already has a live write gate") + +var gateRegistry = struct { + mu sync.Mutex + gates map[*sqlitemigration.Pool]*writeGate +}{gates: map[*sqlitemigration.Pool]*writeGate{}} + +func registerWriteGate(pool *sqlitemigration.Pool, g *writeGate) error { + gateRegistry.mu.Lock() + defer gateRegistry.mu.Unlock() + if _, exists := gateRegistry.gates[pool]; exists { + return errGateExists + } + gateRegistry.gates[pool] = g + return nil +} + +func unregisterWriteGate(pool *sqlitemigration.Pool, g *writeGate) { + gateRegistry.mu.Lock() + defer gateRegistry.mu.Unlock() + if gateRegistry.gates[pool] == g { + delete(gateRegistry.gates, pool) + } +} + +// gateForPool returns the pool's live gate, or nil. +func gateForPool(pool *sqlitemigration.Pool) *writeGate { + gateRegistry.mu.Lock() + defer gateRegistry.mu.Unlock() + return gateRegistry.gates[pool] +} + +// writeStatement is one INSERT/UPDATE/DELETE prepared on a pool connection. +// seq orders it against the gate's Close (writeGate.owns). +type writeStatement struct { + pool *sqlitemigration.Pool + conn *sqlite.Conn + op sqlite.OpType + table string + seq uint64 +} + +var writeStmtSeq atomic.Uint64 + +// writeStmtObserver receives every write statement prepared on any pool +// connection; nil in production. Tests install the AC-G1 ledger here. +var writeStmtObserver atomic.Pointer[func(writeStatement)] + +// writeGateObserver receives every gate at construction; nil in production. +var writeGateObserver atomic.Pointer[func(*writeGate)] + +// noteWriteStatement is the write authorizer's sink (sqlite.go). +func noteWriteStatement(pool *sqlitemigration.Pool, conn *sqlite.Conn, op sqlite.OpType, table string) { + if strings.HasPrefix(table, "sqlite_") { + // Schema migration bookkeeping (CREATE/ALTER TABLE write sqlite_master) + // runs on the migration connection at pool open, before any traffic. + return + } + rec := writeStatement{pool: pool, conn: conn, op: op, table: table, seq: writeStmtSeq.Add(1)} + if g := gateForPool(pool); g != nil && !g.owns(conn, rec.seq) { + metrics.IncSqliteUngatedWrite(writeOpLabel(op), table) + } + if obs := writeStmtObserver.Load(); obs != nil { + (*obs)(rec) + } +} + +func writeOpLabel(op sqlite.OpType) string { + switch op { + case sqlite.OpInsert: + return "insert" + case sqlite.OpUpdate: + return "update" + case sqlite.OpDelete: + return "delete" + } + return "other" +} diff --git a/pkg/registry/file/writegate_tg1_test.go b/pkg/registry/file/writegate_tg1_test.go new file mode 100644 index 000000000..117959fde --- /dev/null +++ b/pkg/registry/file/writegate_tg1_test.go @@ -0,0 +1,188 @@ +package file + +// T-G1 of .omc/plans/write-gate-sharing.md: one table-driven test per +// enumerated write site outside the gate (§1, W1–W9b), run under the flag-on +// topology with the statement recorder on every pool connection and state +// seeded only through the fixture handle. Each site asserts (i) zero +// INSERT/UPDATE/DELETE on any connection the gate never owned, (ii) the +// site's own statement on a gate-owned connection, and (iii) — once, for the +// statements themselves — that the delete/read use the primary key. +// +// On the prototype topology before gate sharing (R0) every site is red at +// (i): that failing run is the evidence the bug class exists there. After R1 +// every site is green. + +import ( + "context" + "strings" + "testing" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/storage" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitex" +) + +type tg1Write struct { + op sqlite.OpType + table string +} + +type tg1Site struct { + name string + state string // an acg2States name, "" for nothing seeded + run func(e *acg2Env, key string) error + // want is the statement the site must run on a gate-owned connection. + want tg1Write +} + +func acg2StateByName(t *testing.T, name string) acg2KeyState { + t.Helper() + for _, st := range acg2States { + if st.name == name { + return st + } + } + t.Fatalf("no key state %q", name) + return acg2KeyState{} +} + +func tg1Update(input runtime.Object, _ storage.ResponseMeta) (runtime.Object, *uint64, error) { + m := input.(metav1.Object) + labels := m.GetLabels() + if labels == nil { + labels = map[string]string{} + } + labels["tg1"] = "updated" + m.SetLabels(labels) + return input, nil, nil +} + +var tg1Sites = []tg1Site{ + {name: "W1-create", run: func(e *acg2Env, key string) error { + obj := newSBOM().(*softwarecomposition.SBOMSyft) + _, _, _, _, _, obj.Name = K8sPathToKeys(key) + return e.legacy.Create(e.ctx, key, obj, nil, 0) + }, want: tg1Write{sqlite.OpInsert, "metadata"}}, + {name: "W1-update", state: "present", run: func(e *acg2Env, key string) error { + return e.legacy.GuaranteedUpdate(e.ctx, key, &softwarecomposition.SBOMSyft{}, false, nil, tg1Update, nil) + }, want: tg1Write{sqlite.OpInsert, "metadata"}}, + {name: "W3-Delete", state: "present", run: func(e *acg2Env, key string) error { + return e.legacy.Delete(e.ctx, key, &softwarecomposition.SBOMSyft{}, nil, nil, nil, storage.DeleteOptions{}) + }, want: tg1Write{sqlite.OpDelete, "metadata"}}, + {name: "W3-DeleteWithConn", state: "present", run: func(e *acg2Env, key string) error { + conn, err := e.pool.Take(e.ctx) + if err != nil { + return err + } + defer e.pool.Put(conn) + return e.legacy.DeleteWithConn(e.ctx, conn, key, &softwarecomposition.SBOMSyft{}, nil, nil, nil, storage.DeleteOptions{}) + }, want: tg1Write{sqlite.OpDelete, "metadata"}}, + {name: "W4-orphan-prune", state: "orphan", run: func(e *acg2Env, key string) error { + return e.legacy.Get(e.ctx, key, storage.GetOptions{}, &softwarecomposition.SBOMSyft{}) + }, want: tg1Write{sqlite.OpDelete, "metadata"}}, + {name: "W5-corrupt-delete", state: "corrupt", run: func(e *acg2Env, key string) error { + return e.legacy.Get(e.ctx, key, storage.GetOptions{}, &softwarecomposition.SBOMSyft{}) + }, want: tg1Write{sqlite.OpDelete, "metadata"}}, + {name: "W6a-migrate-toolfails", state: "wrongtype-toolfails", run: tg1GetHoldingWriteLock, want: tg1Write{sqlite.OpDelete, "metadata"}}, + {name: "W6b-migrate-toolsucceeds", state: "wrongtype-toolsucceeds", run: tg1GetHoldingWriteLock, want: tg1Write{sqlite.OpInsert, "metadata"}}, + {name: "W7a-migrateUnlocked-toolfails", state: "wrongtype-toolfails", run: func(e *acg2Env, key string) error { + return e.legacy.Get(e.ctx, key, storage.GetOptions{}, &softwarecomposition.SBOMSyft{}) + }, want: tg1Write{sqlite.OpDelete, "metadata"}}, + {name: "W7b-migrateUnlocked-toolsucceeds", state: "wrongtype-toolsucceeds", run: func(e *acg2Env, key string) error { + return e.legacy.Get(e.ctx, key, storage.GetOptions{}, &softwarecomposition.SBOMSyft{}) + }, want: tg1Write{sqlite.OpInsert, "metadata"}}, + {name: "W8-list-rewrite-toolsucceeds", state: "wrongtype-toolsucceeds", run: func(e *acg2Env, key string) error { + return e.legacy.GetByNamespace(e.ctx, acg2Group, acg2Kind, acg2DefaultNS, &softwarecomposition.SBOMSyftList{}) + }, want: tg1Write{sqlite.OpInsert, "metadata"}}, + {name: "W9a-cleanup-delete", state: "present-unreferenced", run: tg1CleanupTick, want: tg1Write{sqlite.OpDelete, "metadata"}}, + {name: "W9b-cleanup-migrate", state: "file-without-row", run: tg1CleanupTick, want: tg1Write{sqlite.OpInsert, "metadata"}}, +} + +// tg1GetHoldingWriteLock reaches get()'s hasWriteLock state (migrateObject, +// W6) the way GuaranteedUpdateWithConn does: under Lock(key). +func tg1GetHoldingWriteLock(e *acg2Env, key string) error { + conn, err := e.pool.Take(e.ctx) + if err != nil { + return err + } + defer e.pool.Put(conn) + if err := e.legacy.locks.Lock(e.ctx, key); err != nil { + return err + } + defer e.legacy.locks.Unlock(key) + return e.legacy.get(e.ctx, conn, key, storage.GetOptions{}, &softwarecomposition.SBOMSyft{}, hasWriteLock) +} + +func tg1CleanupTick(e *acg2Env, _ string) error { + return e.cleanup.CleanupTask(e.ctx, map[string][]TypeCleanupHandlerFunc{acg2Kind: {deleteByImageId}}) +} + +func TestTG1_EveryLegacyWriteSiteIsGated(t *testing.T) { + installACG2MigrationTool(t) + for _, site := range tg1Sites { + site := site + t.Run(site.name, func(t *testing.T) { + t.Parallel() + e := newACG2OnEnv(t) + key := sbomKey(e, strings.ToLower(site.name)) + if site.state != "" { + obj := newSBOM() + _, _, _, _, _, name := K8sPathToKeys(key) + obj.(metav1.Object).SetName(name) + acg2StateByName(t, site.state).seed(e, key, obj) + } + mark := e.rec.mark() + err := site.run(e, key) + actions := e.rec.since(mark) + t.Logf("%s: err=%v", site.name, err) + + now := writeStmtSeq.Load() + var ungated []string + var gated []tg1Write + for _, a := range actions { + if !isWriteOp(a.op) || a.table == "" { + continue + } + if e.gate.owns(a.conn, now) { + gated = append(gated, tg1Write{a.op, a.table}) + } else { + ungated = append(ungated, writeOpLabel(a.op)+" "+a.table) + } + } + assert.Empty(t, ungated, "%s: write statements prepared on a connection the gate does not own (the bug class)", site.name) + assert.Contains(t, gated, site.want, "%s: expected %s on %s on the gate's connection; gated writes: %v", site.name, writeOpLabel(site.want.op), site.want.table, gated) + }) + } +} + +// TestTG1_WriteStatementsUsePrimaryKey (iii): the legacy delete and the +// CAS read both resolve the row through the metadata primary key, so a hold +// is one index seek (PM-G1's index case). +func TestTG1_WriteStatementsUsePrimaryKey(t *testing.T) { + e := newACG2OnEnv(t) + plan := func(sql string) string { + var details []string + require.NoError(t, sqlitex.ExecuteTransient(e.fixture, "EXPLAIN QUERY PLAN "+sql, &sqlitex.ExecOptions{ + Args: []any{"k", "n", "x"}, + ResultFunc: func(stmt *sqlite.Stmt) error { + details = append(details, stmt.ColumnText(3)) + return nil + }, + })) + return strings.Join(details, "; ") + } + for _, sql := range []string{ + `DELETE FROM metadata WHERE kind = ? AND namespace = ? AND name = ? RETURNING metadata`, + `SELECT metadata FROM metadata WHERE kind = ? AND namespace = ? AND name = ?`, + } { + p := plan(sql) + assert.True(t, strings.Contains(p, "sqlite_autoindex_metadata_1") || strings.Contains(p, "PRIMARY KEY"), "%s: plan %q does not use the primary key", sql, p) + } +} + +var _ = context.Background diff --git a/pkg/registry/file/writegate_tg2_test.go b/pkg/registry/file/writegate_tg2_test.go new file mode 100644 index 000000000..0b7c7f19a --- /dev/null +++ b/pkg/registry/file/writegate_tg2_test.go @@ -0,0 +1,319 @@ +package file + +// T-G2, T-G5 and T-G6 of .omc/plans/write-gate-sharing.md: the gate's +// re-entrancy detectors, the cold-path pool bound and shutdown ordering. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "testing" + "time" + + helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/storage" + "k8s.io/component-base/metrics/legacyregistry" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitex" +) + +func seedOrphanSBOM(e *acg2Env, key string) { + obj := newSBOM() + _, _, _, _, _, name := K8sPathToKeys(key) + obj.(metav1.Object).SetName(name) + acg2StateByName(e.t, "orphan").seed(e, key, obj) +} + +// TestTG2_ReentrantAcquireThroughThreadedCtxFailsFast: a gated fn that +// re-enters the store through the ctx it was handed (a repair on an orphan +// key, W4) is refused at O(1) on the ctx marker — under the test binary as a +// panic the outer transaction recovers into an error — instead of queuing +// behind its own holder for the whole request deadline. +func TestTG2_ReentrantAcquireThroughThreadedCtxFailsFast(t *testing.T) { + e := newACG2OnEnv(t) + key := sbomKey(e, "tg2-threaded") + seedOrphanSBOM(e, key) + + start := time.Now() + err := e.legacy.write(e.ctx, nil, priorityLow, "test", "test", false, func(ctx context.Context, _ *sqlite.Conn) error { + return e.legacy.Get(ctx, key, storage.GetOptions{}, &softwarecomposition.SBOMSyft{}) + }) + elapsed := time.Since(start) + require.Error(t, err) + assert.Contains(t, err.Error(), "re-entrant", "the nested acquire must be refused on the marker, got: %v", err) + assert.Less(t, elapsed, acg2Prompt, "the refusal is O(1), not a queue wait") + assert.False(t, e.gate.held(), "the outer transaction released the gate") + assert.Zero(t, e.gate.watchdogFired.Load()) +} + +// TestTG2_ReentrantAcquireThroughCapturedCtxIsCaughtByWatchdog: the marker +// cannot see a closure that captured the pre-ticket ctx; that nested acquire +// queues behind its own holder until the ctx expires, and the watchdog is +// what names it — the hold outlives the threshold and the holder is logged +// with every goroutine's stack. +func TestTG2_ReentrantAcquireThroughCapturedCtxIsCaughtByWatchdog(t *testing.T) { + oldInterval, oldThreshold := gateWatchdogInterval, gateWatchdogThreshold + gateWatchdogInterval, gateWatchdogThreshold = 10*time.Millisecond, 100*time.Millisecond + t.Cleanup(func() { gateWatchdogInterval, gateWatchdogThreshold = oldInterval, oldThreshold }) + + e := newACG2OnEnv(t) + key := sbomKey(e, "tg2-captured") + seedOrphanSBOM(e, key) + + const bound = 700 * time.Millisecond + outer, cancel := context.WithTimeout(e.ctx, bound) + defer cancel() + start := time.Now() + err := e.legacy.write(outer, nil, priorityLow, "test", "test", false, func(context.Context, *sqlite.Conn) error { + // The captured ctx carries no marker: the repair queues behind us. + return e.legacy.Get(outer, key, storage.GetOptions{}, &softwarecomposition.SBOMSyft{}) + }) + elapsed := time.Since(start) + // The queued repair gives up when the ctx expires; get() swallows that + // and reports the orphan as not found, which the outer fn returns. + assert.True(t, storage.IsNotFound(err), "got %v", err) + assert.GreaterOrEqual(t, elapsed, bound, "the nested acquire hung until the ctx bound") + assert.GreaterOrEqual(t, e.gate.watchdogFired.Load(), int64(1), "the watchdog must have logged the hold") + assert.False(t, e.gate.held()) +} + +// TestTG5_ColdRepairsUnderSaturatedGateDoNotExhaustThePool (PM-G3): twenty +// readers each GET a distinct orphaned key while the gate is saturated by +// ContainerProfile creates. Every repair queues holding its pool connection +// (§3.3's relaxation); the bound is that this never turns into pool +// exhaustion — zero pool-wait timeouts — and every GET returns inside one +// low-lane turn (≤ highBurstLimit high commits) plus the read. +func TestTG5_ColdRepairsUnderSaturatedGateDoNotExhaustThePool(t *testing.T) { + e := newObjectStoreEnv(t) // pool DefaultPoolSize, busy 5 s, legacy guarded + const readers = 20 + keys := make([]string, readers) + for i := range keys { + keys[i] = K8sKeysToPath("", acg2Group, acg2Kind, "", "kubescape", fmt.Sprintf("tg5-%d", i)) + raw, err := json.Marshal(extractFields(&softwarecomposition.SBOMSyft{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("tg5-%d", i), Namespace: "kubescape"}}, []string{"ObjectMeta", "SchemaVersion"})) + require.NoError(t, err) + require.NoError(t, WriteJSON(e.fixture, keys[i], raw)) + } + + timeoutsBefore := poolWaitTimeouts(t) + stop := make(chan struct{}) + var creator sync.WaitGroup + creator.Add(1) + go func() { + defer creator.Done() + for n := 0; ; n++ { + select { + case <-stop: + return + default: + } + p := e.plain(fmt.Sprintf("sat-%d", n)) + p.UID = "" + _ = e.store.Create(e.ctx, e.key(p.Name), p, nil, 0) + } + }() + // Let the creator saturate the gate before the readers arrive. + require.Eventually(t, func() bool { return e.gate.held() }, 5*time.Second, time.Millisecond) + + var wg sync.WaitGroup + durations := make([]time.Duration, readers) + errs := make([]error, readers) + for i := 0; i < readers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + t0 := time.Now() + errs[i] = e.legacy.Get(e.ctx, keys[i], storage.GetOptions{}, &softwarecomposition.SBOMSyft{}) + durations[i] = time.Since(t0) + }(i) + } + wg.Wait() + close(stop) + creator.Wait() + + assert.Equal(t, timeoutsBefore, poolWaitTimeouts(t), "pool-wait timeouts must stay at zero with the cold paths exercised") + for i := range keys { + assert.True(t, storage.IsNotFound(errs[i]), "reader %d: %v", i, errs[i]) + assert.Less(t, durations[i], 2*time.Second, "reader %d took %s: a queued repair, not a busy-wait", i, durations[i]) + _, err := ReadMetadata(e.fixture, keys[i]) + assert.ErrorIs(t, err, ErrMetadataNotFound, "reader %d: the orphan row was repaired", i) + } + var maxD time.Duration + for _, d := range durations { + if d > maxD { + maxD = d + } + } + t.Logf("T-G5: %d readers under a saturated gate, slowest GET %s, pool-wait timeouts %d", readers, maxD, poolWaitTimeouts(t)-timeoutsBefore) +} + +// poolWaitTimeouts sums storage_pool_wait_duration_seconds{outcome="timeout"} +// over every kind. +func poolWaitTimeouts(t *testing.T) uint64 { + t.Helper() + families, err := legacyregistry.DefaultGatherer.Gather() + require.NoError(t, err) + var n uint64 + for _, mf := range families { + if mf.GetName() != "storage_pool_wait_duration_seconds" { + continue + } + for _, m := range mf.GetMetric() { + for _, l := range m.GetLabel() { + if l.GetName() == "outcome" && l.GetValue() == "timeout" { + n += m.GetHistogram().GetSampleCount() + } + } + } + } + return n +} + +// TestTG6_CloseWithQueuedWritersAndCleanupMidWalk (PM-G7): a legacy commit +// queued on the high lane and a cleanup tick queued on the low lane when the +// gate closes both fail with errGateClosed, the holder finishes, Close +// returns, and the pool closes (the env's cleanup asserts Pool.Close +// returns: every connection is back). +func TestTG6_CloseWithQueuedWritersAndCleanupMidWalk(t *testing.T) { + e := newACG2OnEnv(t) + // The rows live in a non-default namespace: CleanupTask drops the default + // namespace's error (`err = h.cleanupNamespace(...); return nil`, + // cleanup.go), a pre-existing flag-off behaviour this change leaves alone. + for _, n := range []string{"tg6-a", "tg6-b"} { + key := K8sKeysToPath("", acg2Group, acg2Kind, "", "other", n) + obj := &softwarecomposition.SBOMSyft{ObjectMeta: metav1.ObjectMeta{Name: n, Namespace: "other"}} + acg2StateByName(t, "present-unreferenced").seed(e, key, obj) + } + + held := make(chan struct{}) + releaseHolder := make(chan struct{}) + holderDone := make(chan error, 1) + go func() { + holderDone <- e.gate.run(e.ctx, priorityHigh, "tg6-holder", "test", func(context.Context, *sqlite.Conn) error { + close(held) + <-releaseHolder + return nil + }) + }() + <-held + + createErr := make(chan error, 1) + go func() { + obj := newSBOM().(*softwarecomposition.SBOMSyft) + obj.Name = "tg6-create" + createErr <- e.legacy.Create(e.ctx, sbomKey(e, obj.Name), obj, nil, 0) + }() + cleanupErr := make(chan error, 1) + go func() { + cleanupErr <- e.cleanup.CleanupTask(e.ctx, map[string][]TypeCleanupHandlerFunc{acg2Kind: {deleteByImageId}}) + }() + require.Eventually(t, func() bool { h, l := e.gate.queued(); return h == 1 && l == 1 }, 5*time.Second, time.Millisecond, "one writer queued in each lane") + + closed := make(chan struct{}) + go func() { _ = e.gate.Close(); close(closed) }() + select { + case <-closed: + t.Fatal("Close returned while the holder was still inside its transaction") + case <-time.After(50 * time.Millisecond): + } + cErr, clErr := <-createErr, <-cleanupErr + t.Logf("queued create: %v; queued cleanup: %v", cErr, clErr) + assert.ErrorIs(t, cErr, errGateClosed, "the queued legacy commit") + assert.ErrorIs(t, clErr, errGateClosed, "the mid-walk cleanup tick") + close(releaseHolder) + require.NoError(t, <-holderDone) + select { + case <-closed: + case <-time.After(5 * time.Second): + t.Fatal("Close did not return after the holder released") + } + // Reads survive a closed gate. The aborted tick removed the first file + // before its row delete was refused (cleanup's file-then-row order, kept + // as today), so that row is an orphan the next tick repairs. + assert.NoError(t, e.legacy.Get(e.ctx, K8sKeysToPath("", acg2Group, acg2Kind, "", "other", "tg6-b"), storage.GetOptions{ResourceVersion: softwarecomposition.ResourceVersionMetadata}, &softwarecomposition.SBOMSyft{})) +} + +// TestStorageImpl_GateRequiresSingleWriter (RC-6): a StorageImpl sharing the +// gate refuses Create/GuaranteedUpdate while the single-writer path is off +// with a sentinel (errors.Is, not string matching), and keeps serving reads +// and deletes — the signal a test that flipped the package variable gets. +func TestStorageImpl_GateRequiresSingleWriter(t *testing.T) { + e := newACG2OnEnv(t) + key := sbomKey(e, "rc6") + obj := newSBOM() + obj.(metav1.Object).SetName("rc6") + acg2StateByName(t, "present").seed(e, key, obj) + + old := singleWriterEnabled + singleWriterEnabled = false + t.Cleanup(func() { singleWriterEnabled = old }) + + err := e.legacy.Create(e.ctx, sbomKey(e, "rc6-new"), &softwarecomposition.SBOMSyft{ObjectMeta: metav1.ObjectMeta{Name: "rc6-new", Namespace: acg2DefaultNS}}, nil, 0) + assert.ErrorIs(t, err, ErrGateRequiresSingleWriter) + err = e.legacy.GuaranteedUpdate(e.ctx, key, &softwarecomposition.SBOMSyft{}, false, nil, + func(input runtime.Object, _ storage.ResponseMeta) (runtime.Object, *uint64, error) { + return input, nil, nil + }, nil) + assert.ErrorIs(t, err, ErrGateRequiresSingleWriter) + assert.NoError(t, e.legacy.Get(e.ctx, key, storage.GetOptions{}, &softwarecomposition.SBOMSyft{})) + assert.NoError(t, e.legacy.Delete(e.ctx, key, &softwarecomposition.SBOMSyft{}, nil, nil, nil, storage.DeleteOptions{})) +} + +// TestWriteGate_OwnsIsBoundedByTenure: a write recorded on a connection +// BEFORE the gate took it from the pool (a pool taker's write, when it was +// still a pool connection) is not the gate's — owns() is bounded below by +// the sequence at which the gate took the connection, and the ledger judges +// that record ungated. +func TestWriteGate_OwnsIsBoundedByTenure(t *testing.T) { + pool := NewPoolWithOptions(t.TempDir()+"/tenure.sq3", PoolOptions{Size: 1, DisableAutoCheckpoint: true}) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + conn, err := pool.Take(ctx) + require.NoError(t, err) + require.NoError(t, sqlitex.ExecuteTransient(conn, `INSERT INTO metadata (kind,namespace,name,metadata) VALUES ('k','n','pre-tenure','{}')`, nil)) + preSeq := writeStmtSeq.Load() + pool.Put(conn) + + // Pool size 1: the gate takes the very same connection. + g, err := newWriteGate(ctx, pool) + require.NoError(t, err) + require.Same(t, conn, g.conn) + assert.False(t, g.owns(conn, preSeq), "a write before the gate's tenure is not the gate's") + require.NoError(t, g.run(ctx, priorityHigh, "tenure", "test", func(_ context.Context, c *sqlite.Conn) error { + return sqlitex.ExecuteTransient(c, `INSERT INTO metadata (kind,namespace,name,metadata) VALUES ('k','n','in-tenure','{}')`, nil) + })) + assert.True(t, g.owns(conn, writeStmtSeq.Load()), "a write inside the tenure is the gate's") + + // The ledger sees exactly the pre-tenure INSERT as ungated (judging it + // here keeps TestMain's sweep clean). + vs := acg1Ledger.violations(pool) + require.Len(t, vs, 1) + assert.Equal(t, preSeq, vs[0].seq) + assert.Equal(t, sqlite.OpInsert, vs[0].op) + require.NoError(t, g.Close()) + require.NoError(t, pool.Close()) +} + +// TestContainerProfileStorageImpl_RefusesGatedStorageImpl (W11/W12): the +// legacy ContainerProfile storage's long pool-connection transactions must +// never run over a StorageImpl that shares the write gate — its saveObject +// would queue on the gate behind the lock its own connection holds. +func TestContainerProfileStorageImpl_RefusesGatedStorageImpl(t *testing.T) { + e := newACG2OnEnv(t) + cps := NewContainerProfileStorageImpl(e.legacy, e.pool) + ctx, cleanup, err := cps.WithConnection(e.ctx) + require.NoError(t, err) + defer cleanup() + _, err = cps.BeginTransaction(ctx) + assert.ErrorIs(t, err, errCPStorageGated) + assert.ErrorIs(t, cps.HealDivergence(ctx, sbomKey(e, "w12")), errCPStorageGated) +} + +var _ = errors.Is +var _ = helpersv1.ImageIDMetadataKey diff --git a/pkg/registry/softwarecomposition/containerprofile/backend_differential_test.go b/pkg/registry/softwarecomposition/containerprofile/backend_differential_test.go new file mode 100644 index 000000000..2a5f78a3a --- /dev/null +++ b/pkg/registry/softwarecomposition/containerprofile/backend_differential_test.go @@ -0,0 +1,311 @@ +package containerprofile + +// Backend differential suite (the Phase 4 pattern one level down): the SAME +// hand-written CustomREST (genericrest.Store) over the legacy row+gob-file +// StorageImpl and over the SQLite-native ObjectStore +// (pkg/registry/file/sqliteobject_*.go), driven through identical REST +// Create / Get / Update / Delete / List / Watch sequences. Externally +// observable results must be identical except the two codec deltas the +// storage-level suite pins (file.TestDifferential_*): GET creationTimestamp is +// truncated to whole seconds on the new store, and v1beta1 spec collections +// without omitempty come back as [] instead of null. + +import ( + "context" + "path/filepath" + "testing" + "time" + + helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "github.com/kubescape/storage/pkg/config" + "github.com/kubescape/storage/pkg/registry/file" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/apiserver/pkg/registry/rest" + "k8s.io/apiserver/pkg/storage" +) + +// newObjectStoreTestStorage mirrors apiserver.go's wiring under +// ContainerProfileSqliteBackend: the legacy default StorageImpl carries the +// kind-ownership guard and serves GetSbom; the ObjectStore serves the +// containerprofile kind. +func newObjectStoreTestStorage(t *testing.T) storage.Interface { + t.Helper() + dbPath := filepath.Join(t.TempDir(), "metadata.sq3") + pool := file.NewPoolWithOptions(dbPath, file.PoolOptions{DisableAutoCheckpoint: true, BusyTimeout: 5 * time.Second}) + sch := newTestScheme(t) + legacy := file.NewStorageImpl(afero.NewMemMapFs(), file.DefaultStorageRoot, pool, nil, sch) + legacy.(*file.StorageImpl).SetForeignKinds(file.IsContainerProfileKind) + processor := file.NewContainerProfileProcessor(config.Config{DefaultNamespace: "kubescape", MaxContainerProfileSize: 40000}, nil) + processor.Interval = 0 + gateCtx, gateCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer gateCancel() + gate, err := file.NewWriteGate(gateCtx, pool) + require.NoError(t, err) + legacy.(*file.StorageImpl).SetWriteGate(gate) + store, err := file.NewObjectStore(pool, dbPath, nil, sch, processor, legacy, gate, file.ObjectStoreOptions{CheckpointInterval: time.Hour}) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, store.Close()) + require.NoError(t, gate.Close()) + require.NoError(t, pool.Close()) + }) + return store +} + +// backendHarness: the same CustomREST over each backend. +type backendHarness struct { + oldREST rest.StandardStorage // CustomREST over the legacy StorageImpl + newREST rest.StandardStorage // CustomREST over the ObjectStore +} + +func newBackendHarness(t *testing.T) *backendHarness { + t.Helper() + sch := newTestScheme(t) + optsGetter := newOptsGetter() + oldStorage, _ := newTestStorage(t) + oldREST, err := NewCustomREST(sch, oldStorage, optsGetter) + require.NoError(t, err) + newREST, err := NewCustomREST(sch, newObjectStoreTestStorage(t), optsGetter) + require.NoError(t, err) + return &backendHarness{oldREST: oldREST, newREST: newREST} +} + +// canonical strips instance-specific fields (UID, creationTimestamp, checksum) +// and the nil-vs-empty codec delta so the two backends' objects can be +// compared field by field. +func canonical(t *testing.T, obj runtime.Object) *softwarecomposition.ContainerProfile { + t.Helper() + out := normalize(t, obj) + nilifyEmptyCollections(out) + return out +} + +func nilifyEmptyCollections(cp *softwarecomposition.ContainerProfile) { + if len(cp.Spec.Architectures) == 0 { + cp.Spec.Architectures = nil + } + if len(cp.Spec.Capabilities) == 0 { + cp.Spec.Capabilities = nil + } + if len(cp.Spec.Execs) == 0 { + cp.Spec.Execs = nil + } + if len(cp.Spec.Opens) == 0 { + cp.Spec.Opens = nil + } + if len(cp.Spec.Syscalls) == 0 { + cp.Spec.Syscalls = nil + } + if len(cp.Spec.Endpoints) == 0 { + cp.Spec.Endpoints = nil + } + if len(cp.Spec.PolicyByRuleId) == 0 { + cp.Spec.PolicyByRuleId = nil + } + if len(cp.Spec.IdentifiedCallStacks) == 0 { + cp.Spec.IdentifiedCallStacks = nil + } + if len(cp.Spec.Ingress) == 0 { + cp.Spec.Ingress = nil + } + if len(cp.Spec.Egress) == 0 { + cp.Spec.Egress = nil + } + if len(cp.Spec.MatchLabels) == 0 { + cp.Spec.MatchLabels = nil + } + if len(cp.Spec.MatchExpressions) == 0 { + cp.Spec.MatchExpressions = nil + } +} + +func rvOf(t *testing.T, obj runtime.Object) string { + t.Helper() + m, err := meta.Accessor(obj) + require.NoError(t, err) + return m.GetResourceVersion() +} + +func profileWithSpec(name string) *softwarecomposition.ContainerProfile { + return &softwarecomposition.ContainerProfile{ + ObjectMeta: metav1.ObjectMeta{Name: name, Annotations: map[string]string{helpersv1.StatusMetadataKey: helpersv1.Learning}}, + Spec: softwarecomposition.ContainerProfileSpec{ + Architectures: []string{"amd64"}, + // Empty and non-nil on purpose: PreSave keeps it that way, gob drops + // it (GET → nil) and JSON keeps it (GET → []): the codec delta. + Capabilities: []string{}, + Execs: []softwarecomposition.ExecCalls{{Path: "/bin/sh", Args: []string{"/bin/sh"}}}, + ImageTag: "img:1", + }, + } +} + +func TestBackendDifferential_CreateGetUpdateDelete(t *testing.T) { + h := newBackendHarness(t) + ctx := testContext("ns1") + cp := profileWithSpec("cp-a") + + oldOut := mustCreate(t, ctx, h.oldREST, cp) + newOut := mustCreate(t, ctx, h.newREST, cp) + assert.Equal(t, canonical(t, oldOut), canonical(t, newOut)) + assert.Equal(t, rvOf(t, oldOut), rvOf(t, newOut)) + + _, err := h.oldREST.Create(ctx, cp.DeepCopy(), rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) + assert.True(t, apierrors.IsAlreadyExists(err), "old: %v", err) + _, err = h.newREST.Create(ctx, cp.DeepCopy(), rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) + assert.True(t, apierrors.IsAlreadyExists(err), "new: %v", err) + + oldGot, err := h.oldREST.Get(ctx, "cp-a", &metav1.GetOptions{}) + require.NoError(t, err) + newGot, err := h.newREST.Get(ctx, "cp-a", &metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, canonical(t, oldGot), canonical(t, newGot)) + // The codec delta, pinned rather than hidden: the store never persisted + // empty Capabilities, yet GET returns nil on the old store and [] on new. + assert.Nil(t, oldGot.(*softwarecomposition.ContainerProfile).Spec.Capabilities) + assert.NotNil(t, newGot.(*softwarecomposition.ContainerProfile).Spec.Capabilities) + assert.Equal(t, 0, newGot.(*softwarecomposition.ContainerProfile).CreationTimestamp.Nanosecond()) + + update := func(r rest.StandardStorage, mutate func(*softwarecomposition.ContainerProfile)) (runtime.Object, bool, error) { + return r.Update(ctx, "cp-a", rest.DefaultUpdatedObjectInfo(nil, func(_ context.Context, newObj, oldObj runtime.Object) (runtime.Object, error) { + out := oldObj.(*softwarecomposition.ContainerProfile).DeepCopy() + mutate(out) + return out, nil + }), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}) + } + addLabel := func(c *softwarecomposition.ContainerProfile) { + if c.Labels == nil { + c.Labels = map[string]string{} + } + c.Labels["k"] = "v" + } + oldUpd, oldCreated, err := update(h.oldREST, addLabel) + require.NoError(t, err) + newUpd, newCreated, err := update(h.newREST, addLabel) + require.NoError(t, err) + assert.Equal(t, oldCreated, newCreated) + assert.Equal(t, canonical(t, oldUpd), canonical(t, newUpd)) + assert.Equal(t, rvOf(t, oldUpd), rvOf(t, newUpd)) + assert.Equal(t, "2", rvOf(t, newUpd)) + + // no-op update: same RV on both (#315) + oldNoop, _, err := update(h.oldREST, func(*softwarecomposition.ContainerProfile) {}) + require.NoError(t, err) + newNoop, _, err := update(h.newREST, func(*softwarecomposition.ContainerProfile) {}) + require.NoError(t, err) + assert.Equal(t, rvOf(t, oldNoop), rvOf(t, newNoop)) + assert.Equal(t, "2", rvOf(t, newNoop)) + + // stale resourceVersion → Conflict on both + staleUpdate := func(r rest.StandardStorage) error { + _, _, err := r.Update(ctx, "cp-a", rest.DefaultUpdatedObjectInfo(nil, func(_ context.Context, newObj, oldObj runtime.Object) (runtime.Object, error) { + out := oldObj.(*softwarecomposition.ContainerProfile).DeepCopy() + out.ResourceVersion = "1" + out.Labels["k"] = "stale" + return out, nil + }), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}) + return err + } + assert.True(t, apierrors.IsConflict(staleUpdate(h.oldREST))) + assert.True(t, apierrors.IsConflict(staleUpdate(h.newREST))) + + oldDel, oldImmediate, err := h.oldREST.Delete(ctx, "cp-a", rest.ValidateAllObjectFunc, &metav1.DeleteOptions{}) + require.NoError(t, err) + newDel, newImmediate, err := h.newREST.Delete(ctx, "cp-a", rest.ValidateAllObjectFunc, &metav1.DeleteOptions{}) + require.NoError(t, err) + assert.Equal(t, oldImmediate, newImmediate) + assertEqualDeleteStatus(t, oldDel, newDel, "cp-a") + + _, err = h.oldREST.Get(ctx, "cp-a", &metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(err)) + _, err = h.newREST.Get(ctx, "cp-a", &metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(err)) + _, _, err = h.oldREST.Delete(ctx, "cp-a", rest.ValidateAllObjectFunc, &metav1.DeleteOptions{}) + assert.True(t, apierrors.IsNotFound(err)) + _, _, err = h.newREST.Delete(ctx, "cp-a", rest.ValidateAllObjectFunc, &metav1.DeleteOptions{}) + assert.True(t, apierrors.IsNotFound(err)) +} + +func TestBackendDifferential_ListAndContinue(t *testing.T) { + h := newBackendHarness(t) + ctx := testContext("ns1") + for _, n := range []string{"l-a", "l-b", "l-c"} { + mustCreate(t, ctx, h.oldREST, profileWithSpec(n)) + mustCreate(t, ctx, h.newREST, profileWithSpec(n)) + } + names := func(obj runtime.Object) []string { + l := obj.(*softwarecomposition.ContainerProfileList) + var out []string + for _, it := range l.Items { + out = append(out, it.Name) + } + return out + } + for _, r := range []rest.StandardStorage{h.oldREST, h.newREST} { + p1, err := r.List(ctx, &metainternalversion.ListOptions{Limit: 2}) + require.NoError(t, err) + assert.Equal(t, []string{"l-a", "l-b"}, names(p1)) + cont := p1.(*softwarecomposition.ContainerProfileList).Continue + require.NotEmpty(t, cont) + p2, err := r.List(ctx, &metainternalversion.ListOptions{Limit: 2, Continue: cont}) + require.NoError(t, err) + assert.Equal(t, []string{"l-c"}, names(p2)) + full, err := r.List(ctx, &metainternalversion.ListOptions{ResourceVersion: softwarecomposition.ResourceVersionFullSpec}) + require.NoError(t, err) + require.Len(t, full.(*softwarecomposition.ContainerProfileList).Items, 3) + assert.Equal(t, "img:1", full.(*softwarecomposition.ContainerProfileList).Items[0].Spec.ImageTag, "fullSpec list carries the spec") + } + oldL, _ := h.oldREST.List(ctx, &metainternalversion.ListOptions{}) + newL, _ := h.newREST.List(ctx, &metainternalversion.ListOptions{}) + require.Len(t, oldL.(*softwarecomposition.ContainerProfileList).Items, 3) + for i := range oldL.(*softwarecomposition.ContainerProfileList).Items { + o := &oldL.(*softwarecomposition.ContainerProfileList).Items[i] + n := &newL.(*softwarecomposition.ContainerProfileList).Items[i] + assert.Equal(t, canonical(t, o), canonical(t, n)) + } +} + +func TestBackendDifferential_Watch(t *testing.T) { + h := newBackendHarness(t) + ctx := testContext("ns1") + events := func(r rest.StandardStorage) []string { + w, err := r.Watch(ctx, &metainternalversion.ListOptions{ResourceVersion: softwarecomposition.ResourceVersionFullSpec}) + require.NoError(t, err) + defer w.Stop() + mustCreate(t, ctx, r, profileWithSpec("w-a")) + _, _, err = r.Update(ctx, "w-a", rest.DefaultUpdatedObjectInfo(nil, func(_ context.Context, _, oldObj runtime.Object) (runtime.Object, error) { + out := oldObj.(*softwarecomposition.ContainerProfile).DeepCopy() + out.Spec.ImageTag = "img:2" + return out, nil + }), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}) + require.NoError(t, err) + _, _, err = r.Delete(ctx, "w-a", rest.ValidateAllObjectFunc, &metav1.DeleteOptions{}) + require.NoError(t, err) + var out []string + for len(out) < 3 { + select { + case ev := <-w.ResultChan(): + cp := ev.Object.(*softwarecomposition.ContainerProfile) + out = append(out, string(ev.Type)+" "+cp.Name+" rv="+cp.ResourceVersion+" img="+cp.Spec.ImageTag) + case <-time.After(2 * time.Second): + t.Fatalf("watch events missing after %v", out) + } + } + return out + } + oldEv := events(h.oldREST) + newEv := events(h.newREST) + assert.Equal(t, oldEv, newEv) + assert.Equal(t, string(watch.Added)+" w-a rv=1 img=img:1", newEv[0]) + assert.Equal(t, string(watch.Modified)+" w-a rv=2 img=img:2", newEv[1]) + assert.Equal(t, string(watch.Deleted)+" w-a rv=2 img=", newEv[2], "Deleted carries the metadata-only object on both") +} diff --git a/pkg/utils/mutex.go b/pkg/utils/mutex.go index 2279576af..96484852e 100644 --- a/pkg/utils/mutex.go +++ b/pkg/utils/mutex.go @@ -4,10 +4,50 @@ import ( "context" "errors" "sync" + "sync/atomic" ) var ContextNilError = errors.New("context is nil") +// Lock observer labels: the mode of a MapMutex acquisition and its outcome. +const ( + LockModeWrite = "lock" + LockModeRead = "rlock" + LockOutcomeAcquired = "acquired" + LockOutcomeTimeout = "timeout" +) + +// lockObserver is a nil-in-production hook reporting every Lock/RLock +// attempt's mode and outcome. It exists for the storage work-budget test +// (pkg/registry/file, TestWorkBudget), which needs a read/write axis the +// storage_lock_wait_duration_seconds histogram does not have; production +// pays one atomic load and a nil check per acquisition and allocates nothing. +var lockObserver atomic.Pointer[func(mode, outcome string)] + +// SetLockObserver installs f as the process-wide lock observer; nil uninstalls +// it. The observer is called after the MapMutex's internal mutex is released, +// never under it, and may be called from any goroutine. +func SetLockObserver(f func(mode, outcome string)) { + if f == nil { + lockObserver.Store(nil) + return + } + lockObserver.Store(&f) +} + +func observeLock(mode, outcome string) { + if f := lockObserver.Load(); f != nil { + (*f)(mode, outcome) + } +} + +func lockOutcome(err error) string { + if err != nil { + return LockOutcomeTimeout + } + return LockOutcomeAcquired +} + type keyState struct { cond *sync.Cond readers int @@ -109,14 +149,17 @@ func (m *MapMutex[T]) Lock(ctx context.Context, key T) error { if !s.writer && s.readers == 0 && s.pendingWriters == 0 { s.writer = true m.mu.Unlock() + observeLock(LockModeWrite, LockOutcomeAcquired) return nil } s.pendingWriters++ - return m.lockSlow(ctx, key, s, + err := m.lockSlow(ctx, key, s, func() bool { return !s.writer && s.readers == 0 }, func() { s.pendingWriters--; s.writer = true }, func() { s.pendingWriters-- }, ) + observeLock(LockModeWrite, lockOutcome(err)) + return err } func (m *MapMutex[T]) RLock(ctx context.Context, key T) error { @@ -128,13 +171,16 @@ func (m *MapMutex[T]) RLock(ctx context.Context, key T) error { if !s.writer && s.pendingWriters == 0 { s.readers++ m.mu.Unlock() + observeLock(LockModeRead, LockOutcomeAcquired) return nil } - return m.lockSlow(ctx, key, s, + err := m.lockSlow(ctx, key, s, func() bool { return !s.writer && s.pendingWriters == 0 }, func() { s.readers++ }, nil, ) + observeLock(LockModeRead, lockOutcome(err)) + return err } func (m *MapMutex[T]) Unlock(key T) {