feat(storage): full-ACID ContainerProfile backend with write-gate-sharing

.github/workflows/perf-ab.yaml

@@ -0,0 +1,83 @@
+name: perf-ab
+
+# Tier B of the storage measurement harness (hack/perf-ab.sh): a paired A/B of
+# the branch against its merge-base, rebuilt in the same run on the same
+# runner. GitHub runners are noisy, so INCONCLUSIVE is expected often here and
+# is reported, not hidden; a local run on a quiet machine is the authoritative
+# one. Only a REGRESSION verdict fails the check.
+
+on:
+  workflow_dispatch:
+    inputs:
+      base:
+        description: "Commit to compare against (default: merge-base with origin/main)"
+        required: false
+        type: string
+      pairs:
+        description: "Interleaved (base, head) rounds"
+        required: false
+        type: string
+        default: "10"
+  schedule:
+    - cron: "17 3 * * *"
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+
+jobs:
+  perf-ab:
+    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'perf')
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+      - name: Resolve base
+        id: base
+        run: |
+          if [ -n "${{ inputs.base }}" ]; then
+            echo "sha=${{ inputs.base }}" >> "$GITHUB_OUTPUT"
+          elif [ "${{ github.event_name }}" = "schedule" ]; then
+            echo "sha=$(git rev-list -n 1 --before='1 day ago' origin/main)" >> "$GITHUB_OUTPUT"
+          else
+            echo "sha=$(git merge-base origin/main HEAD)" >> "$GITHUB_OUTPUT"
+          fi
+      - name: Paired A/B
+        id: ab
+        env:
+          BASE: ${{ steps.base.outputs.sha }}
+          PAIRS: ${{ inputs.pairs || '10' }}
+          PERF_AB_OUT_DIR: ${{ runner.temp }}/perf-ab
+          PERF_AB_ALLOW_NOISY: "1"
+        run: |
+          set +e
+          make perf-ab
+          echo "rc=$?" >> "$GITHUB_OUTPUT"
+      - name: Report verdict
+        if: always()
+        env:
+          OUT: ${{ runner.temp }}/perf-ab
+          RC: ${{ steps.ab.outputs.rc }}
+        run: |
+          {
+            echo "## perf-ab: $(cat "$OUT/verdict.txt" 2>/dev/null || echo 'no verdict (driver failed)')"
+            echo
+            echo "base \`${{ steps.base.outputs.sha }}\` vs head \`${{ github.sha }}\`, exit $RC (0 pass, 1 regression, 2 config mismatch, 3 inconclusive, 4 underpowered)"
+            echo
+            if [ -f "$OUT/verdict-table.txt" ]; then echo '```'; cat "$OUT/verdict-table.txt"; echo '```'; fi
+            if [ -f "$OUT/schedule.txt" ]; then echo '<details><summary>schedule</summary>'; echo; echo '```'; cat "$OUT/schedule.txt"; echo '```'; echo '</details>'; fi
+          } >> "$GITHUB_STEP_SUMMARY"
+      - name: Upload rounds
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: perf-ab-${{ github.run_id }}
+          path: ${{ runner.temp }}/perf-ab
+          if-no-files-found: ignore
+      - name: Fail on regression
+        if: steps.ab.outputs.rc == '1'
+        run: exit 1

CONTRIBUTING.md

@@ -3,3 +3,16 @@
 The Kubescape project manages this document in the central project repository.
 
 Go to the [centralized CONTRIBUTING.md](https://github.com/kubescape/project-governance/blob/main/CONTRIBUTING.md)
+
+## Storage hot-path changes
+
+A PR touching `pkg/registry/file/{storage,singlewriter,containerprofile_*,sqlite}.go`
+must carry two numbers (see `docs/features/storage-measurement-harness.md`):
+
+- **Work budgets (Tier A).** `go test ./...` includes `TestWorkBudget`. If the
+  golden `pkg/registry/file/testdata/workbudget.golden.json` changes, the
+  commit message states each row's delta and why; the reviewer checks the
+  attribution.
+- **Paired A/B (Tier B).** Run `make perf-ab` on a quiet machine and quote its
+  verdict line verbatim in the PR description. `INCONCLUSIVE` and
+  `UNDERPOWERED` are not `PASS`.

Makefile

@@ -5,14 +5,21 @@ TAG?=test
 IMAGE?=quay.io/kubescape/$(BINARY_NAME)
 
 
-.PHONY: build test docker-build docker-push
+.PHONY: build test perf-ab docker-build docker-push
 
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o $(BINARY_NAME)
 
 test:
 	go test ./...
 
+# Paired A/B of HEAD against its merge-base on this machine (Tier B of the
+# storage measurement harness). Mandatory before SHIP for any change under
+# pkg/registry/file/{storage,singlewriter,containerprofile_*,sqlite}.go; quote
+# its verdict line in the PR. BASE=<sha> and PAIRS=<n> override the defaults.
+perf-ab:
+	hack/perf-ab.sh
+
 docker-build:
 	docker buildx build --platform linux/amd64 -t $(IMAGE):$(TAG) --load -f $(DOCKERFILE_PATH) .
 docker-push:

build/Dockerfile

@@ -9,6 +9,11 @@ RUN --mount=target=. \
     --mount=type=cache,target=/go/pkg \
     GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /out/migration ./cmd/migration/main.go
 
+RUN --mount=target=. \
+    --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg \
+    GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /out/cpexport ./cmd/cpexport/main.go
+
 RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg \
@@ -18,6 +23,7 @@ FROM gcr.io/distroless/static-debian13:nonroot
 
 COPY --from=builder /out/storage /usr/bin/storage
 COPY --from=builder /out/migration /usr/bin/migration
+COPY --from=builder /out/cpexport /usr/bin/cpexport
 ARG image_version
 ENV RELEASE=$image_version
 

cmd/cpexport/main.go

@@ -0,0 +1,65 @@
+package main
+
+// cpexport writes every ContainerProfile the SQLite-native backend
+// (config.ContainerProfileSqliteBackend) holds in the payloads table back
+// as the legacy gob payload file an older storage binary expects. It is the
+// first half of the rollback order of
+// .omc/plans/full-acid-storage-architecture.md §8.4 — export, THEN
+// downgrade — because an older binary's get() deletes the metadata row of
+// any ContainerProfile whose file is missing, destroying the object.
+//
+// Run it with the storage server stopped (scale the deployment to 0, then
+// run it against the PVC), never alongside a serving process: a write that
+// lands after a key was exported leaves the old binary a stale file.
+//
+//	cpexport [-db /data/metadata.sq3] [-root /data] [-dry-run]
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/kubescape/storage/pkg/apis/softwarecomposition/install"
+	"github.com/kubescape/storage/pkg/registry/file"
+	"github.com/spf13/afero"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+func main() {
+	root := flag.String("root", file.DefaultStorageRoot, "storage root directory holding the payload files")
+	db := flag.String("db", "", "SQLite database path (default <root>/metadata.sq3)")
+	dryRun := flag.Bool("dry-run", false, "decode and count without writing any file")
+	batch := flag.Int("batch", file.DefaultMigrationBatchSize, "rows read per query")
+	flag.Parse()
+	if *db == "" {
+		*db = filepath.Join(*root, "metadata.sq3")
+	}
+	// The pool retries an unopenable database every 5 s until ctx expires
+	// (sqlitemigration.Pool.Take); a missing database is always a wrong
+	// -db/-root, not something to wait an hour for.
+	if _, err := os.Stat(*db); err != nil {
+		fmt.Fprintf(os.Stderr, "cpexport: database: %v\n", err)
+		os.Exit(1)
+	}
+
+	sch := runtime.NewScheme()
+	install.Install(sch)
+	pool := file.NewPoolWithOptions(*db, file.PoolOptions{Size: 2, BusyTimeout: file.DefaultBusyTimeout})
+	defer func() { _ = pool.Close() }()
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Hour)
+	defer cancel()
+	report, err := file.ExportContainerProfiles(ctx, pool, afero.NewOsFs(), *root, sch, file.ContainerProfileExportOptions{BatchSize: *batch, DryRun: *dryRun})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "cpexport: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf("cpexport: exported=%d legacySkipped=%d undecodable=%d dryRun=%v elapsed=%s\n",
+		report.Exported, report.LegacySkipped, report.Undecodable, report.DryRun, report.Elapsed)
+	if report.Undecodable > 0 {
+		os.Exit(2)
+	}
+}