diff --git a/CLAUDE.md b/CLAUDE.md index 3c747e9..d6bf56f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,9 +42,11 @@ Sprue is the upload coordination service for Storacha local development. It rout **Stores (pkg/store/)** - Each domain has its own store interface in `pkg/store//` -- Each store has two implementations: AWS (DynamoDB/S3) in `/aws/` and in-memory in `/memory/` +- Each store has three implementations: AWS (DynamoDB/S3) in `/aws/`, PostgreSQL (+ S3 for blob payloads) in `/postgres/`, and in-memory in `/memory/` - Store interfaces: `agent.Store`, `blob_registry.Store`, `consumer.Store`, `customer.Store`, `delegation.Store`, `metrics.Store`, `replica.Store`, `revocation.Store`, `space_diff.Store`, `storage_provider.Store`, `subscription.Store`, `upload.Store` -- AWS stores are wired in `internal/fx/store/aws/provider.go`, memory stores in `internal/fx/store/memory/provider.go` +- Backends are wired in `internal/fx/store//provider.go` (aws, postgres, memory) +- Backend selection is driven by `storage.type` in config (`memory` | `postgres` | `aws`; default `postgres`). Per-backend settings live under `storage.postgres`, `storage.dynamodb`, and `storage.s3`. +- Postgres schema is managed by goose migrations in `internal/migrations/sql/`, embedded and applied on startup. Set `storage.postgres.skip_migrations: true` to disable. **Services (pkg/)** - `provisioning`: Manages space provisioning (consumers + subscriptions) @@ -66,18 +68,21 @@ Sprue is the upload coordination service for Storacha local development. It rout ### Configuration -Configuration via YAML file or environment variables with `UPLOAD_` prefix: -- `UPLOAD_SERVER_HOST`, `UPLOAD_SERVER_PORT` -- `UPLOAD_IDENTITY_KEY_FILE`, `UPLOAD_IDENTITY_PRIVATE_KEY`, `UPLOAD_IDENTITY_SERVICE_DID` -- `UPLOAD_PIRI_ENDPOINT`, `UPLOAD_INDEXER_ENDPOINT` -- `UPLOAD_DYNAMODB_*` for DynamoDB settings - -Legacy env vars without prefix (e.g., `HOST`, `PORT`, `KEY_FILE`) also supported. +Configuration via YAML file or environment variables with `SPRUE_` prefix: +- `SPRUE_STORAGE_TYPE` — selects the store backend (`memory`, `postgres`, `aws`; default `postgres`) +- `SPRUE_SERVER_HOST`, `SPRUE_SERVER_PORT` +- `SPRUE_IDENTITY_KEY_FILE`, `SPRUE_IDENTITY_PRIVATE_KEY`, `SPRUE_IDENTITY_SERVICE_DID` +- `SPRUE_INDEXER_ENDPOINT` +- `SPRUE_STORAGE_POSTGRES_DSN`, `SPRUE_STORAGE_POSTGRES_MAX_CONNS`, `SPRUE_STORAGE_POSTGRES_SKIP_MIGRATIONS` +- `SPRUE_STORAGE_DYNAMODB_*` for DynamoDB settings (AWS backend) +- `SPRUE_STORAGE_S3_*` for S3/MinIO settings ### Key Dependencies - **go-ucanto**: UCAN RPC framework for capability-based authorization - **go-libstoracha**: Storacha capability definitions (blob, space, upload, etc.) - **echo/v4**: HTTP server framework -- **aws-sdk-go-v2**: DynamoDB client +- **aws-sdk-go-v2**: DynamoDB + S3 client +- **jackc/pgx/v5**: PostgreSQL driver +- **pressly/goose/v3**: SQL schema migrations - **viper/cobra**: Configuration and CLI diff --git a/Makefile b/Makefile index b4fbf76..546faa3 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ VERSION=$(shell awk -F'"' '/"version":/ {print $$4}' version.json) COMMIT=$(shell git rev-parse --short HEAD) DATE=$(shell date -u -Iseconds) GOFLAGS=-ldflags="-X github.com/storacha/sprue/pkg/build.version=$(VERSION) -X github.com/storacha/sprue/pkg/build.Commit=$(COMMIT) -X github.com/storacha/sprue/pkg/build.Date=$(DATE) -X github.com/storacha/sprue/pkg/build.BuiltBy=make" +DOCKER := $(shell which docker) .PHONY: all build test lint clean docker-build @@ -35,4 +36,4 @@ clean: rm -f ./sprue docker-build: - docker build -t sprue:latest . + $(DOCKER) build -t sprue:latest . diff --git a/README.md b/README.md index 98d6c4f..354d313 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,30 @@ # Sprue -The Storacha upload service in Go. +The Forge upload service in Go (formerly the Storacha upload service). + +## Running locally + +The repo ships a `docker-compose.yaml` that brings up sprue alongside +PostgreSQL and MinIO for self-hosted development: + +```bash +docker compose up -d postgres minio +SPRUE_STORAGE_POSTGRES_DSN="postgres://sprue:sprue@localhost:5432/sprue?sslmode=disable" \ + ./sprue serve +``` + +Postgres is the default store backend, so no extra flag is required. + +## Store backends + +Sprue supports three store backends, selected by +`storage.type` (or `SPRUE_STORAGE_TYPE`; defaults to `postgres`): + +- `memory` — in-process only; all data is lost on restart. Dev/test only. +- `postgres` — PostgreSQL for metadata + S3-compatible storage (MinIO, Ceph, AWS S3) + for blob payloads. Schema is managed by goose migrations embedded in + `internal/migrations/sql/` and applied on startup. +- `aws` — DynamoDB for metadata + S3 for blob payloads. ## Notes diff --git a/config.example.yaml b/config.example.yaml index 4af7623..7d4f3c5 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1,12 +1,13 @@ # Sprue Configuration # # This file documents all configuration options. Copy to config.yaml and customize. -# All values can also be set via environment variables with UPLOAD_ prefix. +# All values can also be set via environment variables with SPRUE_ prefix. # Environment variables take precedence over config file values. # # Examples: -# server.host -> UPLOAD_SERVER_HOST -# dynamodb.endpoint -> UPLOAD_DYNAMODB_ENDPOINT +# server.host -> SPRUE_SERVER_HOST +# storage.type -> SPRUE_STORAGE_TYPE +# storage.postgres.dsn -> SPRUE_STORAGE_POSTGRES_DSN # # Deployment configuration @@ -20,11 +21,6 @@ deployment: # includes the original blob that was uploaded, so only values above 1 will # allow users to have multiple copies of their data. max_replicas: 3 - # Indicates whether to use in-memory stores instead of DynamoDB/S3. All data - # will be lost on service restart when this is true, so it should only be used - # for development or testing. It overrides all other store-related config when - # true. - in_memory_stores: false server: # Host address to bind the HTTP server to @@ -63,33 +59,51 @@ mailer: # Secret for CRAMMD5 SMTP authentication smtp_auth_secret: "" -dynamodb: - # DynamoDB endpoint (for local development with DynamoDB Local) - endpoint: "http://dynamodb-local:8000" - # AWS region for DynamoDB - region: "us-west-1" - agent_index_table: "agent-index" - blob_registry_table: "blob-registry" - consumer_table: "consumer" - customer_table: "customer" - delegation_table: "delegation" - space_metrics_table: "space-metrics" - admin_metrics_table: "admin-metrics" - replica_table: "replica" - revocation_table: "revocation" - storage_provider_table: "storage-provider" - subscription_table: "subscription" - space_diff_table: "space-diff" - upload_table: "upload" +# Storage backend selection and per-backend configuration. +storage: + # Selects the backend. Valid values: "memory", "postgres", "aws". Default "postgres". + # - memory: in-process stores — all data lost on restart; dev/test only. + # - postgres: PostgreSQL for metadata + S3-compatible storage for blob payloads. + # - aws: DynamoDB for metadata + S3 for blob payloads. + type: "postgres" -s3: - # S3 endpoint (for local development with MinIO) - endpoint: "http://minio:9000" - # S3 region - region: "us-west-1" - agent_message_bucket: "agent-message" - delegation_bucket: "delegation" - upload_shards_bucket: "upload-shards" + postgres: + # libpq-style connection string (used when storage.type is "postgres"). + dsn: "postgres://sprue:sprue@postgres:5432/sprue?sslmode=disable" + # Maximum number of pool connections. 0 uses the pgx default. + max_conns: 10 + # Minimum idle connections to keep warm. + min_conns: 0 + # Set to true to skip goose migrations at startup (they run by default). + skip_migrations: false + + dynamodb: + # DynamoDB endpoint (for local development with DynamoDB Local). + endpoint: "http://dynamodb-local:8000" + # AWS region for DynamoDB. + region: "us-west-1" + agent_index_table: "agent-index" + blob_registry_table: "blob-registry" + consumer_table: "consumer" + customer_table: "customer" + delegation_table: "delegation" + space_metrics_table: "space-metrics" + admin_metrics_table: "admin-metrics" + replica_table: "replica" + revocation_table: "revocation" + storage_provider_table: "storage-provider" + subscription_table: "subscription" + space_diff_table: "space-diff" + upload_table: "upload" + + s3: + # S3 endpoint (for local development with MinIO). + endpoint: "http://minio:9000" + # S3 region. + region: "us-west-1" + agent_message_bucket: "agent-message" + delegation_bucket: "delegation" + upload_shards_bucket: "upload-shards" log: # Log level: debug, info, warn, error diff --git a/go.mod b/go.mod index 6715a82..6da5e59 100644 --- a/go.mod +++ b/go.mod @@ -13,23 +13,41 @@ require ( github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-log/v2 v2.9.0 github.com/ipld/go-ipld-prime v0.21.1-0.20240917223228-6148356a4c2e + github.com/jackc/pgx/v5 v5.9.1 github.com/labstack/echo/v4 v4.14.0 github.com/multiformats/go-multiaddr v0.16.0 github.com/multiformats/go-multibase v0.2.0 github.com/multiformats/go-multihash v0.2.3 github.com/olekukonko/tablewriter v0.0.5 + github.com/pressly/goose/v3 v3.27.0 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 github.com/storacha/go-libstoracha v0.7.5 github.com/storacha/go-ucanto v0.7.2 github.com/stretchr/testify v1.11.1 - github.com/testcontainers/testcontainers-go v0.41.0 + github.com/testcontainers/testcontainers-go v0.42.0 github.com/testcontainers/testcontainers-go/modules/dynamodb v0.41.0 github.com/testcontainers/testcontainers-go/modules/minio v0.40.0 + github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 go.uber.org/fx v1.24.0 go.uber.org/zap v1.27.0 ) +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/mfridman/interpolate v0.0.2 // indirect + github.com/moby/moby/api v1.54.1 // indirect + github.com/moby/moby/client v0.4.0 // indirect + github.com/moby/sys/atomicwriter v0.1.0 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect + golang.org/x/sync v0.20.0 // indirect +) + require ( dario.cat/mergo v1.0.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect @@ -59,7 +77,6 @@ require ( github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect - github.com/creack/pty v1.1.24 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/distribution/reference v0.6.0 // indirect @@ -78,7 +95,6 @@ require ( github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect @@ -99,7 +115,7 @@ require ( github.com/ipld/go-car v0.6.2 // indirect github.com/ipld/go-codec-dagpb v1.6.0 // indirect github.com/ipni/go-libipni v0.6.18 // indirect - github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/compress v1.18.5 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect @@ -112,12 +128,11 @@ require ( github.com/minio/sha256-simd v1.0.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect - github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect github.com/moby/sys/sequential v0.6.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect - github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect @@ -136,8 +151,8 @@ require ( github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect - github.com/shirou/gopsutil/v4 v4.26.2 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect + github.com/shirou/gopsutil/v4 v4.26.3 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.15.0 // indirect @@ -153,24 +168,22 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect - go.opentelemetry.io/otel v1.42.0 // indirect - go.opentelemetry.io/otel/metric v1.42.0 // indirect - go.opentelemetry.io/otel/trace v1.42.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.19.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.48.0 // indirect - golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa // indirect - golang.org/x/net v0.51.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.41.0 // indirect - golang.org/x/text v0.34.0 // indirect + golang.org/x/crypto v0.49.0 // indirect + golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.35.0 // indirect golang.org/x/time v0.14.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260316172706-e463d84ca32d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260311181403-84a4fc48630c // indirect - google.golang.org/grpc v1.79.2 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.4.1 // indirect diff --git a/go.sum b/go.sum index 319eb39..7dc4c21 100644 --- a/go.sum +++ b/go.sum @@ -278,6 +278,7 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= @@ -371,6 +372,14 @@ github.com/ipld/go-ipld-prime v0.21.1-0.20240917223228-6148356a4c2e h1:0Anxx6pMS github.com/ipld/go-ipld-prime v0.21.1-0.20240917223228-6148356a4c2e/go.mod h1:LN+1Tx6867lbDCmf8bErp1TNw3Kh9eY2n0eJ+whRx38= github.com/ipni/go-libipni v0.6.18 h1:x8X6y0QoMmSKtwRlczWdWEYedoLUGCEek2TttfDKPk4= github.com/ipni/go-libipni v0.6.18/go.mod h1:qUObcCVXMx3byEGn/g2alGlsqY79tTZBzWoNPCwYFOE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= @@ -386,8 +395,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= -github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/koron/go-ssdp v0.0.5 h1:E1iSMxIs4WqxTbIBLtmNBeOOC+1sCIXQeqTWVnpmwhk= @@ -404,6 +413,8 @@ github.com/labstack/echo/v4 v4.14.0 h1:+tiMrDLxwv6u0oKtD03mv+V1vXXB3wCqPHJqPuIe+ github.com/labstack/echo/v4 v4.14.0/go.mod h1:xmw1clThob0BSVRX1CRQkGQ/vjwcpOMjQZSZa9fKA/c= github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-libp2p v0.41.1 h1:8ecNQVT5ev/jqALTvisSJeVNvXYJyK4NhQx1nNRXQZE= @@ -432,6 +443,10 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= +github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= +github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= +github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= @@ -451,8 +466,12 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= -github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= -github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4= +github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw= +github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= @@ -494,6 +513,8 @@ github.com/multiformats/go-varint v0.1.0 h1:i2wqFp4sdl3IcIxfAonHQV9qU5OsZ4Ts9IOo github.com/multiformats/go-varint v0.1.0/go.mod h1:5KVAVXegtfmNQQm/lCY+ATvDzvJJhSkUlGQV9wgObdI= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -560,6 +581,8 @@ github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a/go.mod h1:ocZfO/ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/pressly/goose/v3 v3.27.0 h1:/D30gVTuQhu0WsNZYbJi4DMOsx1lNq+6SkLe+Wp59BM= +github.com/pressly/goose/v3 v3.27.0/go.mod h1:3ZBeCXqzkgIRvrEMDkYh1guvtoJTU5oMMuDdkutoM78= github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -567,14 +590,16 @@ github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/quic-go/quic-go v0.50.1 h1:unsgjFIUqW8a2oopkY7YNONpV1gYND6Nt9hnt1PN94Q= github.com/quic-go/quic-go v0.50.1/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -587,15 +612,17 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI= -github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= +github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= +github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -628,8 +655,8 @@ github.com/storacha/go-libstoracha v0.7.5/go.mod h1:htUh/VZ0qHRLPJKWZsgXv9mCOqlA github.com/storacha/go-ucanto v0.7.2 h1:sLg+swDM/6VEcrb9VOik3hP8ek3NvqqKWiZRmsva5X0= github.com/storacha/go-ucanto v0.7.2/go.mod h1:DZlWyzuSkXk3phAuJpGDyhxYWpJogW1RFqp/VfldT64= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -641,12 +668,14 @@ github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/testcontainers/testcontainers-go v0.41.0 h1:mfpsD0D36YgkxGj2LrIyxuwQ9i2wCKAD+ESsYM1wais= -github.com/testcontainers/testcontainers-go v0.41.0/go.mod h1:pdFrEIfaPl24zmBjerWTTYaY0M6UHsqA1YSvsoU40MI= +github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= +github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= github.com/testcontainers/testcontainers-go/modules/dynamodb v0.41.0 h1:/ft73PYqMamsp+LabfY08SJZBZMAp9B8GLnk8IYgg0A= github.com/testcontainers/testcontainers-go/modules/dynamodb v0.41.0/go.mod h1:1e9/q6+o2MuaMRXf5tzePUG2O89PQfiS47dTcsKwddM= github.com/testcontainers/testcontainers-go/modules/minio v0.40.0 h1:M+Ib1mIXq/hEcH8tyEvBnOZ7NJi03zY+P1gYO5GGp6o= github.com/testcontainers/testcontainers-go/modules/minio v0.40.0/go.mod h1:ON0MxxS/pME0SJOKLImw/D9R1L7apYsxIZrM/uEqORA= +github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 h1:GCbb1ndrF7OTDiIvxXyItaDab4qkzTFJ48LKFdM7EIo= +github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0/go.mod h1:IRPBaI8jXdrNfD0e4Zm7Fbcgaz5shKxOQv4axiL09xs= github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= @@ -688,22 +717,22 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= -go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= -go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0 h1:inYW9ZhgqiDqh6BioM7DVHHzEGVq76Db5897WLGZ5Go= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0/go.mod h1:Izur+Wt8gClgMJqO/cZ8wdeeMryJ/xxiOVgFSSfpDTY= -go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= -go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= -go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= -go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= -go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= -go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= -go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= -go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= -go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= -go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= @@ -736,8 +765,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -748,8 +777,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4= -golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk= +golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= +golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -778,8 +807,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= -golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -819,8 +848,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= -golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -896,18 +925,17 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -918,8 +946,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -982,8 +1010,8 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= -golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= -golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1060,10 +1088,10 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto/googleapis/api v0.0.0-20260316172706-e463d84ca32d h1:RdWlPmVySdTF0IBIZzvZJvSD0ZocPBNUsnE+uGBxj+4= -google.golang.org/genproto/googleapis/api v0.0.0-20260316172706-e463d84ca32d/go.mod h1:X2gu9Qwng7Nn009s/r3RUxqkzQNqOrAy79bluY7ojIg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260311181403-84a4fc48630c h1:xgCzyF2LFIO/0X2UAoVRiXKU5Xg6VjToG4i2/ecSswk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260311181403-84a4fc48630c/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1084,8 +1112,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= -google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1128,6 +1156,16 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= +modernc.org/libc v1.68.0 h1:PJ5ikFOV5pwpW+VqCK1hKJuEWsonkIJhhIXyuF/91pQ= +modernc.org/libc v1.68.0/go.mod h1:NnKCYeoYgsEqnY3PgvNgAeaJnso968ygU8Z0DxjoEc0= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.46.1 h1:eFJ2ShBLIEnUWlLy12raN0Z1plqmFX9Qe3rjQTKt6sU= +modernc.org/sqlite v1.46.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/config/config.go b/internal/config/config.go index 85da944..66ac223 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,14 +7,20 @@ import ( "github.com/spf13/viper" ) +// Valid values for StorageConfig.Type. +const ( + StorageTypeMemory = "memory" + StorageTypePostgres = "postgres" + StorageTypeAWS = "aws" +) + // Config holds the sprue service configuration. type Config struct { Deployment DeploymentConfig `mapstructure:"deployment"` Server ServerConfig `mapstructure:"server"` Identity IdentityConfig `mapstructure:"identity"` Indexer IndexerConfig `mapstructure:"indexer"` - DynamoDB DynamoDBConfig `mapstructure:"dynamodb"` - S3 S3Config `mapstructure:"s3"` + Storage StorageConfig `mapstructure:"storage"` Log LogConfig `mapstructure:"log"` Mailer MailerConfig `mapstructure:"mailer"` } @@ -30,11 +36,6 @@ type DeploymentConfig struct { // given blob. It includes the original blob that was uploaded, so only values // above 1 will allow users to have multiple copies of their data. MaxReplicas uint `mapstructure:"max_replicas"` - // InMemoryStores indicates whether to use in-memory stores instead of - // DynamoDB/S3. All data will be lost on service restart when this is true, so - // it should only be used for development or testing. It overrides all other - // store-related config when true. - InMemoryStores bool `mapstructure:"in_memory_stores"` } // ServerConfig holds HTTP server settings. @@ -71,6 +72,24 @@ type IndexerConfig struct { DID string `mapstructure:"did"` } +// StorageConfig selects and configures the store backend. Type picks which of +// Memory/Postgres/DynamoDB to use; S3 is shared by the postgres and aws +// backends for blob payload storage. +type StorageConfig struct { + // Type selects the backend: "memory", "postgres", or "aws". Defaults to + // "postgres". + Type string `mapstructure:"type"` + + Memory MemoryConfig `mapstructure:"memory"` + Postgres PostgresConfig `mapstructure:"postgres"` + DynamoDB DynamoDBConfig `mapstructure:"dynamodb"` + S3 S3Config `mapstructure:"s3"` +} + +// MemoryConfig configures the in-process store. It currently carries no +// settings but exists for symmetry with the persistent backends. +type MemoryConfig struct{} + // DynamoDBConfig holds DynamoDB settings. type DynamoDBConfig struct { // Endpoint is the DynamoDB endpoint (for local development). @@ -94,6 +113,20 @@ type DynamoDBConfig struct { UploadTable string `mapstructure:"upload_table"` } +// PostgresConfig holds PostgreSQL settings. +type PostgresConfig struct { + // DSN is a libpq-style connection string, e.g. + // "postgres://user:pass@host:5432/db?sslmode=disable". + DSN string `mapstructure:"dsn"` + // MaxConns is the maximum number of connections the pool will hold. + MaxConns int32 `mapstructure:"max_conns"` + // MinConns is the minimum number of idle connections the pool maintains. + MinConns int32 `mapstructure:"min_conns"` + // SkipMigrations disables automatic goose migrations on startup. Default + // (false) runs migrations. + SkipMigrations bool `mapstructure:"skip_migrations"` +} + // S3Config holds S3 settings. type S3Config struct { // Endpoint is the S3 endpoint (for local development). @@ -140,36 +173,37 @@ func SetDefaults(v *viper.Viper) { // Indexer defaults (port 80 for did:web resolution in Docker) v.SetDefault("indexer.endpoint", "http://indexer:80") - // DynamoDB defaults - v.SetDefault("dynamodb.endpoint", "http://dynamodb-local:8000") - v.SetDefault("dynamodb.region", "us-west-1") - v.SetDefault("dynamodb.provider_table", "delegator-provider-info") - v.SetDefault("dynamodb.allocations_table", "upload-allocations") - v.SetDefault("dynamodb.receipts_table", "upload-receipts") - v.SetDefault("dynamodb.auth_requests_table", "upload-auth-requests") - v.SetDefault("dynamodb.provisionings_table", "upload-provisionings") - v.SetDefault("dynamodb.uploads_table", "upload-uploads") - - v.SetDefault("dynamodb.agent_index_table", "agent-index") - v.SetDefault("dynamodb.blob_registry_table", "blob-registry") - v.SetDefault("dynamodb.consumer_table", "consumer") - v.SetDefault("dynamodb.customer_table", "customer") - v.SetDefault("dynamodb.delegation_table", "delegation") - v.SetDefault("dynamodb.space_metrics_table", "space-metrics") - v.SetDefault("dynamodb.admin_metrics_table", "admin-metrics") - v.SetDefault("dynamodb.replica_table", "replica") - v.SetDefault("dynamodb.revocation_table", "revocation") - v.SetDefault("dynamodb.storage_provider_table", "storage-provider") - v.SetDefault("dynamodb.subscription_table", "subscription") - v.SetDefault("dynamodb.space_diff_table", "space-diff") - v.SetDefault("dynamodb.upload_table", "upload") - - // S3 defaults - v.SetDefault("s3.endpoint", "http://minio:9000") - v.SetDefault("s3.region", "us-west-1") - v.SetDefault("s3.agent_message_bucket", "agent-message") - v.SetDefault("s3.delegation_bucket", "delegation") - v.SetDefault("s3.upload_shards_bucket", "upload-shards") + // Storage defaults — Postgres is the default backend. + v.SetDefault("storage.type", StorageTypePostgres) + + // Postgres defaults + v.SetDefault("storage.postgres.dsn", "postgres://sprue:sprue@postgres:5432/sprue?sslmode=disable") + v.SetDefault("storage.postgres.max_conns", 10) + v.SetDefault("storage.postgres.min_conns", 0) + + // DynamoDB defaults (only consulted when storage.type is "aws") + v.SetDefault("storage.dynamodb.endpoint", "http://dynamodb-local:8000") + v.SetDefault("storage.dynamodb.region", "us-west-1") + v.SetDefault("storage.dynamodb.agent_index_table", "agent-index") + v.SetDefault("storage.dynamodb.blob_registry_table", "blob-registry") + v.SetDefault("storage.dynamodb.consumer_table", "consumer") + v.SetDefault("storage.dynamodb.customer_table", "customer") + v.SetDefault("storage.dynamodb.delegation_table", "delegation") + v.SetDefault("storage.dynamodb.space_metrics_table", "space-metrics") + v.SetDefault("storage.dynamodb.admin_metrics_table", "admin-metrics") + v.SetDefault("storage.dynamodb.replica_table", "replica") + v.SetDefault("storage.dynamodb.revocation_table", "revocation") + v.SetDefault("storage.dynamodb.storage_provider_table", "storage-provider") + v.SetDefault("storage.dynamodb.subscription_table", "subscription") + v.SetDefault("storage.dynamodb.space_diff_table", "space-diff") + v.SetDefault("storage.dynamodb.upload_table", "upload") + + // S3 defaults (used by the postgres and aws backends) + v.SetDefault("storage.s3.endpoint", "http://minio:9000") + v.SetDefault("storage.s3.region", "us-west-1") + v.SetDefault("storage.s3.agent_message_bucket", "agent-message") + v.SetDefault("storage.s3.delegation_bucket", "delegation") + v.SetDefault("storage.s3.upload_shards_bucket", "upload-shards") // Log defaults v.SetDefault("log.level", "info") @@ -185,32 +219,8 @@ func BindEnvVars(v *viper.Viper) { // Load creates a viper instance and loads configuration from the given config file // (if provided), environment variables, and defaults. func Load(configFile string) (*Config, error) { - v := viper.New() - - SetDefaults(v) - BindEnvVars(v) - - if configFile != "" { - v.SetConfigFile(configFile) - if err := v.ReadInConfig(); err != nil { - return nil, fmt.Errorf("reading config file: %w", err) - } - } else { - // Look for config in standard locations - v.SetConfigName("config") - v.SetConfigType("yaml") - v.AddConfigPath(".") - v.AddConfigPath("/etc/sprue/") - // Ignore error if no config file found - use defaults and env vars - _ = v.ReadInConfig() - } - - var cfg Config - if err := v.Unmarshal(&cfg); err != nil { - return nil, fmt.Errorf("unmarshaling config: %w", err) - } - - return &cfg, nil + cfg, _, err := LoadWithViper(configFile) + return cfg, err } // LoadWithViper creates a viper instance and loads configuration, returning both @@ -231,6 +241,7 @@ func LoadWithViper(configFile string) (*Config, *viper.Viper, error) { v.SetConfigType("yaml") v.AddConfigPath(".") v.AddConfigPath("/etc/sprue/") + // Ignore error if no config file found - use defaults and env vars _ = v.ReadInConfig() } diff --git a/internal/fx/app.go b/internal/fx/app.go index bff5936..0629579 100644 --- a/internal/fx/app.go +++ b/internal/fx/app.go @@ -1,11 +1,14 @@ package fx import ( + "fmt" + "github.com/storacha/sprue/internal/config" "github.com/storacha/sprue/internal/fx/service" "github.com/storacha/sprue/internal/fx/service/handlers" "github.com/storacha/sprue/internal/fx/store/aws" "github.com/storacha/sprue/internal/fx/store/memory" + "github.com/storacha/sprue/internal/fx/store/postgres" "go.uber.org/fx" ) @@ -22,10 +25,17 @@ var AppModule = func(cfg *config.Config) fx.Option { handlers.Module, ServerModule, } - if cfg.Deployment.InMemoryStores { + switch cfg.Storage.Type { + case config.StorageTypeMemory: opts = append(opts, memory.Module) - } else { + case config.StorageTypePostgres, "": + // Empty Type is treated as the default backend (postgres) so callers + // constructing a Config literal in tests don't have to set it. + opts = append(opts, postgres.Module) + case config.StorageTypeAWS: opts = append(opts, aws.Module) + default: + return fx.Error(fmt.Errorf("unknown storage.type %q (valid: memory, postgres, aws)", cfg.Storage.Type)) } return fx.Options(opts...) } diff --git a/internal/fx/app_test.go b/internal/fx/app_test.go index 765cd92..a7b2c31 100644 --- a/internal/fx/app_test.go +++ b/internal/fx/app_test.go @@ -51,29 +51,85 @@ func TestWireApp(t *testing.T) { Indexer: config.IndexerConfig{ Endpoint: "http://localhost:3000", }, - DynamoDB: config.DynamoDBConfig{ - Region: "us-east-1", - Endpoint: dynamoEndpoint.String(), - AgentIndexTable: "agent-index-" + appID, - BlobRegistryTable: "blob-registry-" + appID, - ConsumerTable: "consumer-" + appID, - CustomerTable: "customer-" + appID, - DelegationTable: "delegation-" + appID, - SpaceMetricsTable: "space-metrics-" + appID, - AdminMetricsTable: "admin-metrics-" + appID, - ReplicaTable: "replica-" + appID, - RevocationTable: "revocation-" + appID, - StorageProviderTable: "storage-provider-" + appID, - SubscriptionTable: "subscription-" + appID, - SpaceDiffTable: "space-diff-" + appID, - UploadTable: "upload-" + appID, - }, - S3: config.S3Config{ - Region: "us-east-1", - Endpoint: s3Endpoint.String(), - AgentMessageBucket: "agent-message-" + appID, - DelegationBucket: "delegation-" + appID, - UploadShardsBucket: "upload-shards-" + appID, + Storage: config.StorageConfig{ + Type: config.StorageTypeAWS, + DynamoDB: config.DynamoDBConfig{ + Region: "us-east-1", + Endpoint: dynamoEndpoint.String(), + AgentIndexTable: "agent-index-" + appID, + BlobRegistryTable: "blob-registry-" + appID, + ConsumerTable: "consumer-" + appID, + CustomerTable: "customer-" + appID, + DelegationTable: "delegation-" + appID, + SpaceMetricsTable: "space-metrics-" + appID, + AdminMetricsTable: "admin-metrics-" + appID, + ReplicaTable: "replica-" + appID, + RevocationTable: "revocation-" + appID, + StorageProviderTable: "storage-provider-" + appID, + SubscriptionTable: "subscription-" + appID, + SpaceDiffTable: "space-diff-" + appID, + UploadTable: "upload-" + appID, + }, + S3: config.S3Config{ + Region: "us-east-1", + Endpoint: s3Endpoint.String(), + AgentMessageBucket: "agent-message-" + appID, + DelegationBucket: "delegation-" + appID, + UploadShardsBucket: "upload-shards-" + appID, + }, + }, + Mailer: config.MailerConfig{ + Type: "nop", + }, + Log: config.LogConfig{ + Level: "debug", + }, + } + }, + }, + { + name: "postgres", + configure: func(t *testing.T) config.Config { + if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { + if !testutil.IsDockerAvailable(t) { + t.Fatalf("docker is expected in CI linux testing environments, but wasn't found") + } + } + if !testutil.IsDockerAvailable(t) { + t.SkipNow() + } + pool := testutil.CreatePostgres(t) + s3Endpoint := testutil.CreateS3(t) + appID := uuid.NewString() + + return config.Config{ + Deployment: config.DeploymentConfig{ + Environment: "test", + }, + Server: config.ServerConfig{ + Host: "localhost", + Port: 0, + }, + Identity: config.IdentityConfig{ + PrivateKey: testutil.Must(ed25519.Format(testutil.WebService))(t), + ServiceDID: testutil.WebService.DID().String(), + }, + Indexer: config.IndexerConfig{ + Endpoint: "http://localhost:3000", + }, + Storage: config.StorageConfig{ + Type: config.StorageTypePostgres, + Postgres: config.PostgresConfig{ + DSN: pool.Config().ConnString(), + SkipMigrations: true, // testutil.CreatePostgres already migrated + }, + S3: config.S3Config{ + Region: "us-east-1", + Endpoint: s3Endpoint.String(), + AgentMessageBucket: "agent-message-" + appID, + DelegationBucket: "delegation-" + appID, + UploadShardsBucket: "upload-shards-" + appID, + }, }, Mailer: config.MailerConfig{ Type: "nop", @@ -89,8 +145,7 @@ func TestWireApp(t *testing.T) { configure: func(t *testing.T) config.Config { return config.Config{ Deployment: config.DeploymentConfig{ - Environment: "test", - InMemoryStores: true, + Environment: "test", }, Server: config.ServerConfig{ Host: "localhost", @@ -103,6 +158,9 @@ func TestWireApp(t *testing.T) { Indexer: config.IndexerConfig{ Endpoint: "http://localhost:3000", }, + Storage: config.StorageConfig{ + Type: config.StorageTypeMemory, + }, Mailer: config.MailerConfig{ Type: "nop", }, diff --git a/internal/fx/config.go b/internal/fx/config.go index 2f59d37..011f7b5 100644 --- a/internal/fx/config.go +++ b/internal/fx/config.go @@ -16,21 +16,27 @@ type Configs struct { Server config.ServerConfig Identity config.IdentityConfig Indexer config.IndexerConfig + Storage config.StorageConfig DynamoDB config.DynamoDBConfig + Postgres config.PostgresConfig S3 config.S3Config Log config.LogConfig Mailer config.MailerConfig } -// ProvideConfigs provides the individual fields of the config. +// ProvideConfigs provides the individual fields of the config. Inner storage +// configs (Postgres, DynamoDB, S3) are surfaced flat so store providers can +// consume them directly without knowing about the Storage discriminator. func ProvideConfigs(cfg *config.Config) Configs { return Configs{ Deployment: cfg.Deployment, Server: cfg.Server, Identity: cfg.Identity, Indexer: cfg.Indexer, - DynamoDB: cfg.DynamoDB, - S3: cfg.S3, + Storage: cfg.Storage, + DynamoDB: cfg.Storage.DynamoDB, + Postgres: cfg.Storage.Postgres, + S3: cfg.Storage.S3, Log: cfg.Log, Mailer: cfg.Mailer, } diff --git a/internal/fx/store/postgres/provider.go b/internal/fx/store/postgres/provider.go new file mode 100644 index 0000000..38041ac --- /dev/null +++ b/internal/fx/store/postgres/provider.go @@ -0,0 +1,206 @@ +// Package postgres wires the Postgres-backed store implementations into the +// application via uber-go/fx. It mirrors the layout of +// internal/fx/store/aws/provider.go. +package postgres + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/storacha/sprue/internal/config" + "github.com/storacha/sprue/internal/migrations" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/storacha/sprue/pkg/store/agent" + pgagent "github.com/storacha/sprue/pkg/store/agent/postgres" + blobregistry "github.com/storacha/sprue/pkg/store/blob_registry" + pgblobregistry "github.com/storacha/sprue/pkg/store/blob_registry/postgres" + "github.com/storacha/sprue/pkg/store/consumer" + pgconsumer "github.com/storacha/sprue/pkg/store/consumer/postgres" + "github.com/storacha/sprue/pkg/store/customer" + pgcustomer "github.com/storacha/sprue/pkg/store/customer/postgres" + "github.com/storacha/sprue/pkg/store/delegation" + pgdelegation "github.com/storacha/sprue/pkg/store/delegation/postgres" + "github.com/storacha/sprue/pkg/store/metrics" + pgmetrics "github.com/storacha/sprue/pkg/store/metrics/postgres" + "github.com/storacha/sprue/pkg/store/replica" + pgreplica "github.com/storacha/sprue/pkg/store/replica/postgres" + "github.com/storacha/sprue/pkg/store/revocation" + pgrevocation "github.com/storacha/sprue/pkg/store/revocation/postgres" + spacediff "github.com/storacha/sprue/pkg/store/space_diff" + pgspacediff "github.com/storacha/sprue/pkg/store/space_diff/postgres" + storageprovider "github.com/storacha/sprue/pkg/store/storage_provider" + pgstorageprovider "github.com/storacha/sprue/pkg/store/storage_provider/postgres" + "github.com/storacha/sprue/pkg/store/subscription" + pgsubscription "github.com/storacha/sprue/pkg/store/subscription/postgres" + "github.com/storacha/sprue/pkg/store/upload" + pgupload "github.com/storacha/sprue/pkg/store/upload/postgres" + + // Reuse the AWS S3 client constructor for the three stores that keep an S3 half. + awsstore "github.com/storacha/sprue/internal/fx/store/aws" + + "go.uber.org/fx" + "go.uber.org/zap" +) + +var Module = fx.Module("postgres-store", + fx.Provide( + NewPostgresPool, + NewMigratedPool, + // S3 client is still needed for the three stores (agent, delegation, upload) + // that persist blob payloads outside of the database. + awsstore.NewS3Client, + + fx.Annotate(NewAgentStore, fx.As(new(agent.Store))), + fx.Annotate(NewBlobRegistryStore, fx.As(new(blobregistry.Store))), + fx.Annotate(NewConsumerStore, fx.As(new(consumer.Store))), + fx.Annotate(NewCustomerStore, fx.As(new(customer.Store))), + fx.Annotate(NewDelegationStore, fx.As(new(delegation.Store))), + fx.Annotate(NewSpaceMetricsStore, fx.As(fx.Self()), fx.As(new(metrics.SpaceStore))), + fx.Annotate(NewAdminMetricsStore, fx.As(fx.Self()), fx.As(new(metrics.Store))), + fx.Annotate(NewReplicaStore, fx.As(new(replica.Store))), + fx.Annotate(NewRevocationStore, fx.As(new(revocation.Store))), + fx.Annotate(NewSpaceDiffStore, fx.As(fx.Self()), fx.As(new(spacediff.Store))), + fx.Annotate(NewStorageProviderStore, fx.As(new(storageprovider.Store))), + fx.Annotate(NewSubscriptionStore, fx.As(new(subscription.Store))), + fx.Annotate(NewUploadStore, fx.As(new(upload.Store))), + ), +) + +// MigratedPool is a *pgxpool.Pool whose schema is guaranteed to be at the head +// revision of internal/migrations by the time any OnStart hook that depends on +// it runs. Store constructors depend on *MigratedPool rather than +// *pgxpool.Pool so the fx dependency graph orders NewMigratedPool's migration +// hook before every store's Initialize hook. +type MigratedPool struct { + *pgxpool.Pool +} + +// NewPostgresPool creates a pgx connection pool and registers a lifecycle hook +// to close it at shutdown. +func NewPostgresPool(cfg config.PostgresConfig, lc fx.Lifecycle, logger *zap.Logger) (*pgxpool.Pool, error) { + if cfg.DSN == "" { + return nil, errors.New("postgres.dsn is required when store_backend is \"postgres\"") + } + + poolCfg, err := pgxpool.ParseConfig(cfg.DSN) + if err != nil { + return nil, fmt.Errorf("parsing postgres DSN: %w", err) + } + if cfg.MaxConns > 0 { + poolCfg.MaxConns = cfg.MaxConns + } + if cfg.MinConns > 0 { + poolCfg.MinConns = cfg.MinConns + } + + pool, err := pgxpool.NewWithConfig(context.Background(), poolCfg) + if err != nil { + return nil, fmt.Errorf("creating pgx pool: %w", err) + } + + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + if err := pool.Ping(ctx); err != nil { + return fmt.Errorf("pinging postgres: %w", err) + } + logger.Info("connected to postgres", zap.Int32("max_conns", poolCfg.MaxConns)) + return nil + }, + OnStop: func(ctx context.Context) error { + pool.Close() + return nil + }, + }) + + return pool, nil +} + +// NewMigratedPool registers an OnStart hook that runs goose migrations against +// pool, and returns a *MigratedPool wrapper. Because every store constructor +// depends on *MigratedPool, fx resolves this provider (and appends its OnStart +// hook) before any store's Initialize hook is registered — and OnStart hooks +// fire in registration order — so all stores see a fully migrated schema. +// Migrations are skipped when storage.postgres.skip_migrations is true. +func NewMigratedPool(lc fx.Lifecycle, cfg config.PostgresConfig, pool *pgxpool.Pool, logger *zap.Logger) *MigratedPool { + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + if cfg.SkipMigrations { + logger.Info("skipping postgres migrations (storage.postgres.skip_migrations=true)") + return nil + } + logger.Info("running postgres migrations") + return migrations.Up(ctx, pool, logger) + }, + }) + return &MigratedPool{Pool: pool} +} + +func NewAgentStore(lc fx.Lifecycle, mdb *MigratedPool, s3Cfg config.S3Config, s3Client *s3.Client) agent.Store { + store := pgagent.New(mdb.Pool, s3Client, s3Cfg.AgentMessageBucket) + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + return store.Initialize(ctx) + }, + OnStop: func(ctx context.Context) error { + return store.Shutdown(ctx) + }, + }) + return store +} + +func NewBlobRegistryStore(mdb *MigratedPool, consumerStore consumer.Store) blobregistry.Store { + return pgblobregistry.New(mdb.Pool, consumerStore) +} + +func NewConsumerStore(mdb *MigratedPool) consumer.Store { + return pgconsumer.New(mdb.Pool) +} + +func NewCustomerStore(mdb *MigratedPool) customer.Store { + return pgcustomer.New(mdb.Pool) +} + +func NewDelegationStore(lc fx.Lifecycle, mdb *MigratedPool, s3Cfg config.S3Config, s3Client *s3.Client) delegation.Store { + store := pgdelegation.New(mdb.Pool, s3Client, s3Cfg.DelegationBucket) + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + return store.Initialize(ctx) + }, + }) + return store +} + +func NewSpaceMetricsStore(mdb *MigratedPool) *pgmetrics.SpaceStore { + return pgmetrics.NewSpaceStore(mdb.Pool) +} + +func NewAdminMetricsStore(mdb *MigratedPool) *pgmetrics.Store { + return pgmetrics.New(mdb.Pool) +} + +func NewReplicaStore(mdb *MigratedPool) replica.Store { + return pgreplica.New(mdb.Pool) +} + +func NewRevocationStore(mdb *MigratedPool) revocation.Store { + return pgrevocation.New(mdb.Pool) +} + +func NewSpaceDiffStore(mdb *MigratedPool) *pgspacediff.Store { + return pgspacediff.New(mdb.Pool) +} + +func NewStorageProviderStore(mdb *MigratedPool) storageprovider.Store { + return pgstorageprovider.New(mdb.Pool) +} + +func NewSubscriptionStore(mdb *MigratedPool) subscription.Store { + return pgsubscription.New(mdb.Pool) +} + +func NewUploadStore(mdb *MigratedPool) upload.Store { + return pgupload.New(mdb.Pool) +} diff --git a/internal/migrations/migrations.go b/internal/migrations/migrations.go new file mode 100644 index 0000000..bf8f12e --- /dev/null +++ b/internal/migrations/migrations.go @@ -0,0 +1,55 @@ +// Package migrations embeds the sprue Postgres migrations and exposes a runner +// that applies them via goose. +package migrations + +import ( + "context" + "database/sql" + "embed" + "fmt" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" + "go.uber.org/zap" +) + +//go:embed sql/*.sql +var FS embed.FS + +// Up applies all pending migrations embedded in FS to the database behind pool. +func Up(ctx context.Context, pool *pgxpool.Pool, logger *zap.Logger) error { + db := stdlib.OpenDBFromPool(pool) + defer db.Close() + return runUp(ctx, db, logger) +} + +// UpDB is equivalent to Up but takes a *sql.DB directly. Useful in tests where +// a pool is not available. +func UpDB(ctx context.Context, db *sql.DB, logger *zap.Logger) error { + return runUp(ctx, db, logger) +} + +func runUp(ctx context.Context, db *sql.DB, logger *zap.Logger) error { + goose.SetBaseFS(FS) + goose.SetLogger(&zapGooseLogger{logger: logger}) + if err := goose.SetDialect("postgres"); err != nil { + return fmt.Errorf("setting goose dialect: %w", err) + } + if err := goose.UpContext(ctx, db, "sql"); err != nil { + return fmt.Errorf("running goose migrations: %w", err) + } + return nil +} + +type zapGooseLogger struct { + logger *zap.Logger +} + +func (l *zapGooseLogger) Fatalf(format string, v ...interface{}) { + l.logger.Sugar().Fatalf(format, v...) +} + +func (l *zapGooseLogger) Printf(format string, v ...interface{}) { + l.logger.Sugar().Infof(format, v...) +} diff --git a/internal/migrations/sql/00001_init.sql b/internal/migrations/sql/00001_init.sql new file mode 100644 index 0000000..0388775 --- /dev/null +++ b/internal/migrations/sql/00001_init.sql @@ -0,0 +1,11 @@ +-- +goose Up +-- +goose StatementBegin +-- Sprue Postgres schema — tables are added by per-store migrations. +-- This file exists so the embedded FS is non-empty and goose has a baseline. +SELECT 1; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +SELECT 1; +-- +goose StatementEnd diff --git a/internal/migrations/sql/00002_customer.sql b/internal/migrations/sql/00002_customer.sql new file mode 100644 index 0000000..405aa49 --- /dev/null +++ b/internal/migrations/sql/00002_customer.sql @@ -0,0 +1,19 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE customer ( + customer TEXT PRIMARY KEY, + account TEXT, + product TEXT NOT NULL, + details JSONB, + reserved_capacity BIGINT, + inserted_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ +); + +CREATE INDEX customer_account_idx ON customer (account) WHERE account IS NOT NULL; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS customer; +-- +goose StatementEnd diff --git a/internal/migrations/sql/00003_storage_provider.sql b/internal/migrations/sql/00003_storage_provider.sql new file mode 100644 index 0000000..10228d8 --- /dev/null +++ b/internal/migrations/sql/00003_storage_provider.sql @@ -0,0 +1,17 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE storage_provider ( + provider TEXT PRIMARY KEY, + endpoint TEXT NOT NULL, + proof TEXT NOT NULL, + weight INTEGER NOT NULL, + replication_weight INTEGER, + inserted_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS storage_provider; +-- +goose StatementEnd diff --git a/internal/migrations/sql/00004_consumer.sql b/internal/migrations/sql/00004_consumer.sql new file mode 100644 index 0000000..40597b0 --- /dev/null +++ b/internal/migrations/sql/00004_consumer.sql @@ -0,0 +1,20 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE consumer ( + subscription TEXT NOT NULL, + provider TEXT NOT NULL, + consumer TEXT NOT NULL, + customer TEXT NOT NULL, + cause TEXT NOT NULL, + inserted_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (subscription, provider) +); + +CREATE INDEX consumer_space_idx ON consumer (consumer, provider, subscription); +CREATE INDEX consumer_customer_idx ON consumer (customer, subscription, provider); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS consumer; +-- +goose StatementEnd diff --git a/internal/migrations/sql/00005_subscription.sql b/internal/migrations/sql/00005_subscription.sql new file mode 100644 index 0000000..5dd6761 --- /dev/null +++ b/internal/migrations/sql/00005_subscription.sql @@ -0,0 +1,19 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE subscription ( + subscription TEXT NOT NULL, + provider TEXT NOT NULL, + customer TEXT NOT NULL, + cause TEXT NOT NULL, + inserted_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (subscription, provider) +); + +CREATE INDEX subscription_customer_provider_idx + ON subscription (customer, provider, subscription); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS subscription; +-- +goose StatementEnd diff --git a/internal/migrations/sql/00006_metrics.sql b/internal/migrations/sql/00006_metrics.sql new file mode 100644 index 0000000..c96fc58 --- /dev/null +++ b/internal/migrations/sql/00006_metrics.sql @@ -0,0 +1,20 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE admin_metrics ( + name TEXT PRIMARY KEY, + value BIGINT NOT NULL DEFAULT 0 +); + +CREATE TABLE space_metrics ( + space TEXT NOT NULL, + name TEXT NOT NULL, + value BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (space, name) +); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS space_metrics; +DROP TABLE IF EXISTS admin_metrics; +-- +goose StatementEnd diff --git a/internal/migrations/sql/00007_space_diff.sql b/internal/migrations/sql/00007_space_diff.sql new file mode 100644 index 0000000..2ed88df --- /dev/null +++ b/internal/migrations/sql/00007_space_diff.sql @@ -0,0 +1,18 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE space_diff ( + provider TEXT NOT NULL, + space TEXT NOT NULL, + receipt_at TIMESTAMPTZ NOT NULL, + cause TEXT NOT NULL, + subscription TEXT NOT NULL, + delta BIGINT NOT NULL, + inserted_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (provider, space, receipt_at, cause) +); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS space_diff; +-- +goose StatementEnd diff --git a/internal/migrations/sql/00008_replica.sql b/internal/migrations/sql/00008_replica.sql new file mode 100644 index 0000000..5cc8293 --- /dev/null +++ b/internal/migrations/sql/00008_replica.sql @@ -0,0 +1,18 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE replica ( + space TEXT NOT NULL, + digest TEXT NOT NULL, + provider TEXT NOT NULL, + status TEXT NOT NULL, + cause TEXT NOT NULL, + inserted_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (space, digest, provider) +); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS replica; +-- +goose StatementEnd diff --git a/internal/migrations/sql/00009_revocation.sql b/internal/migrations/sql/00009_revocation.sql new file mode 100644 index 0000000..a48faa6 --- /dev/null +++ b/internal/migrations/sql/00009_revocation.sql @@ -0,0 +1,15 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE revocation ( + revoke TEXT NOT NULL, + scope TEXT NOT NULL, + cause TEXT NOT NULL, + inserted_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (revoke, scope) +); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS revocation; +-- +goose StatementEnd diff --git a/internal/migrations/sql/00010_delegation.sql b/internal/migrations/sql/00010_delegation.sql new file mode 100644 index 0000000..21a3ecc --- /dev/null +++ b/internal/migrations/sql/00010_delegation.sql @@ -0,0 +1,19 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE delegation ( + link TEXT PRIMARY KEY, + audience TEXT NOT NULL, + issuer TEXT NOT NULL, + cause TEXT, + expiration BIGINT, + inserted_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX delegation_audience_idx ON delegation (audience, link); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS delegation; +-- +goose StatementEnd diff --git a/internal/migrations/sql/00011_upload.sql b/internal/migrations/sql/00011_upload.sql new file mode 100644 index 0000000..160bfbc --- /dev/null +++ b/internal/migrations/sql/00011_upload.sql @@ -0,0 +1,27 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE upload ( + space TEXT NOT NULL, + root TEXT NOT NULL, + cause TEXT NOT NULL, + inserted_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (space, root) +); + +CREATE INDEX upload_root_idx ON upload (root); + +CREATE TABLE upload_shard ( + space TEXT NOT NULL, + root TEXT NOT NULL, + shard TEXT NOT NULL, + PRIMARY KEY (space, root, shard), + FOREIGN KEY (space, root) REFERENCES upload (space, root) ON DELETE CASCADE +); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS upload_shard; +DROP TABLE IF EXISTS upload; +-- +goose StatementEnd diff --git a/internal/migrations/sql/00012_blob_registry.sql b/internal/migrations/sql/00012_blob_registry.sql new file mode 100644 index 0000000..c2992f5 --- /dev/null +++ b/internal/migrations/sql/00012_blob_registry.sql @@ -0,0 +1,18 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE blob_registry ( + space TEXT NOT NULL, + digest TEXT NOT NULL, + size BIGINT NOT NULL, + cause TEXT NOT NULL, + inserted_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (space, digest) +); + +CREATE INDEX blob_registry_digest_idx ON blob_registry (digest, space); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS blob_registry; +-- +goose StatementEnd diff --git a/internal/migrations/sql/00013_agent_index.sql b/internal/migrations/sql/00013_agent_index.sql new file mode 100644 index 0000000..8ee6aa6 --- /dev/null +++ b/internal/migrations/sql/00013_agent_index.sql @@ -0,0 +1,16 @@ +-- +goose Up +-- +goose StatementBegin +-- agent_index stores the "task + kind -> (root CID @ message CID)" mapping. +-- "kind" is either "in" (an invocation) or "out" (a receipt). +CREATE TABLE agent_index ( + task TEXT NOT NULL, + kind TEXT NOT NULL, + identifier TEXT NOT NULL, + PRIMARY KEY (task, kind) +); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS agent_index; +-- +goose StatementEnd diff --git a/internal/testutil/postgres.go b/internal/testutil/postgres.go new file mode 100644 index 0000000..61cc789 --- /dev/null +++ b/internal/testutil/postgres.go @@ -0,0 +1,54 @@ +package testutil + +import ( + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/storacha/sprue/internal/migrations" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + "github.com/testcontainers/testcontainers-go/wait" + "go.uber.org/zap" +) + +const ( + testPostgresDB = "sprue" + testPostgresUser = "sprue" + testPostgresPass = "sprue" +) + +// CreatePostgres starts a throwaway Postgres container, runs the sprue +// migrations against it, and returns a connection pool. The container is +// cleaned up when the test finishes. +func CreatePostgres(t *testing.T) *pgxpool.Pool { + t.Helper() + + ctx := t.Context() + container, err := tcpostgres.Run(ctx, + "postgres:16-alpine", + tcpostgres.WithDatabase(testPostgresDB), + tcpostgres.WithUsername(testPostgresUser), + tcpostgres.WithPassword(testPostgresPass), + testcontainers.WithWaitStrategy( + wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2). + WithStartupTimeout(30*time.Second), + ), + ) + testcontainers.CleanupContainer(t, container) + require.NoError(t, err) + + dsn, err := container.ConnectionString(ctx, "sslmode=disable") + require.NoError(t, err) + + t.Logf("Postgres DSN: %s", dsn) + + pool, err := pgxpool.New(ctx, dsn) + require.NoError(t, err) + t.Cleanup(pool.Close) + + require.NoError(t, migrations.Up(ctx, pool, zap.NewNop())) + return pool +} diff --git a/pkg/store/agent/agent_test.go b/pkg/store/agent/agent_test.go index e60b5de..edd8888 100644 --- a/pkg/store/agent/agent_test.go +++ b/pkg/store/agent/agent_test.go @@ -22,17 +22,19 @@ import ( "github.com/storacha/sprue/pkg/store/agent" "github.com/storacha/sprue/pkg/store/agent/aws" "github.com/storacha/sprue/pkg/store/agent/memory" + agentpostgres "github.com/storacha/sprue/pkg/store/agent/postgres" "github.com/stretchr/testify/require" ) type StoreKind string const ( - Memory StoreKind = "memory" - AWS StoreKind = "aws" + Memory StoreKind = "memory" + AWS StoreKind = "aws" + Postgres StoreKind = "postgres" ) -var storeKinds = []StoreKind{Memory, AWS} +var storeKinds = []StoreKind{Memory, AWS, Postgres} func makeStore(t *testing.T, k StoreKind) agent.Store { switch k { @@ -40,10 +42,33 @@ func makeStore(t *testing.T, k StoreKind) agent.Store { return memory.New() case AWS: return createAWSStore(t) + case Postgres: + return createPostgresStore(t) } panic("unknown store kind") } +func createPostgresStore(t *testing.T) agent.Store { + if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { + if !testutil.IsDockerAvailable(t) { + t.Fatalf("docker is expected in CI linux testing environments, but wasn't found") + } + } + if !testutil.IsDockerAvailable(t) { + t.SkipNow() + } + pool := testutil.CreatePostgres(t) + s3Endpoint := testutil.CreateS3(t) + s3Client := testutil.NewS3Client(t, s3Endpoint) + + store := agentpostgres.New(pool, s3Client, "agent-message-"+uuid.NewString()) + require.NoError(t, store.Initialize(t.Context())) + t.Cleanup(func() { + require.NoError(t, store.Shutdown(context.Background())) + }) + return store +} + func createAWSStore(t *testing.T) agent.Store { // This test expects docker to be running in linux CI environments and fails if it's not if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { diff --git a/pkg/store/agent/postgres/store.go b/pkg/store/agent/postgres/store.go new file mode 100644 index 0000000..9cbe916 --- /dev/null +++ b/pkg/store/agent/postgres/store.go @@ -0,0 +1,186 @@ +// Package postgres provides a PostgreSQL-backed implementation of agent.Store. +// Metadata indices live in Postgres; message payloads remain in S3 (matching +// the AWS backend). +package postgres + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + cid "github.com/ipfs/go-cid" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/storacha/go-ucanto/core/car" + "github.com/storacha/go-ucanto/core/dag/blockstore" + "github.com/storacha/go-ucanto/core/invocation" + "github.com/storacha/go-ucanto/core/message" + "github.com/storacha/go-ucanto/core/receipt" + "github.com/storacha/sprue/pkg/internal/ipldutil" + "github.com/storacha/sprue/pkg/store/agent" +) + +type Store struct { + pool *pgxpool.Pool + s3 *s3.Client + bucketName string +} + +var _ agent.Store = (*Store)(nil) + +func New(pool *pgxpool.Pool, s3Client *s3.Client, bucketName string) *Store { + return &Store{ + pool: pool, + s3: s3Client, + bucketName: bucketName, + } +} + +func (s *Store) Initialize(ctx context.Context) error { + if _, err := s.s3.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &s.bucketName}); err != nil { + if _, err := s.s3.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: &s.bucketName}); err != nil { + return fmt.Errorf("creating S3 bucket %q: %w", s.bucketName, err) + } + } + return nil +} + +func (s *Store) Shutdown(ctx context.Context) error { + return nil +} + +func (s *Store) GetInvocation(ctx context.Context, task cid.Cid) (invocation.Invocation, error) { + root, bs, err := s.getByTask(ctx, task, "in") + if err != nil { + return nil, fmt.Errorf("getting invocation for task %s: %w", task, err) + } + return invocation.NewInvocationView(cidlink.Link{Cid: root}, bs) +} + +func (s *Store) GetReceipt(ctx context.Context, task cid.Cid) (receipt.AnyReceipt, error) { + root, bs, err := s.getByTask(ctx, task, "out") + if err != nil { + return nil, fmt.Errorf("getting receipt for task %s: %w", task, err) + } + return receipt.NewAnyReceipt(cidlink.Link{Cid: root}, bs) +} + +func (s *Store) getByTask(ctx context.Context, task cid.Cid, kind string) (cid.Cid, blockstore.BlockReader, error) { + var identifier string + err := s.pool.QueryRow(ctx, ` + SELECT identifier FROM agent_index WHERE task = $1 AND kind = $2 + `, task.String(), kind).Scan(&identifier) + if errors.Is(err, pgx.ErrNoRows) { + if kind == "in" { + return cid.Undef, nil, agent.ErrInvocationNotFound + } + return cid.Undef, nil, agent.ErrReceiptNotFound + } + if err != nil { + return cid.Undef, nil, fmt.Errorf("querying agent_index: %w", err) + } + parts := strings.SplitN(identifier, "@", 2) + if len(parts) != 2 { + return cid.Undef, nil, fmt.Errorf("invalid identifier format: %s", identifier) + } + root, err := cid.Parse(parts[0]) + if err != nil { + return cid.Undef, nil, fmt.Errorf("parsing root CID: %w", err) + } + msgRoot, err := cid.Parse(parts[1]) + if err != nil { + return cid.Undef, nil, fmt.Errorf("parsing message root CID: %w", err) + } + + out, err := s.s3.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &s.bucketName, + Key: aws.String(toMessagePath(msgRoot)), + }) + if err != nil { + return cid.Undef, nil, fmt.Errorf("getting message from S3: %w", err) + } + defer out.Body.Close() + + _, blocks, err := car.Decode(out.Body) + if err != nil { + return cid.Undef, nil, fmt.Errorf("decoding CAR: %w", err) + } + bs, err := blockstore.NewBlockStore(blockstore.WithBlocksIterator(blocks)) + if err != nil { + return cid.Undef, nil, fmt.Errorf("creating blockstore: %w", err) + } + return root, bs, nil +} + +// Write uploads the agent message payload to S3 and records every index entry +// in a single atomic INSERT. The payload is written before the index so that a +// partial failure leaves (at worst) an orphan S3 object rather than a dangling +// index pointer to a missing payload. All work runs on the caller's context, +// so cancellation propagates through both the S3 and Postgres calls. +func (s *Store) Write(ctx context.Context, msg message.AgentMessage, index []agent.IndexEntry, source []byte) error { + msgRoot, err := ipldutil.ToCID(msg.Root().Link()) + if err != nil { + return fmt.Errorf("converting message root link to CID: %w", err) + } + + if _, err := s.s3.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &s.bucketName, + Key: aws.String(toMessagePath(msgRoot)), + Body: bytes.NewReader(source), + }); err != nil { + return fmt.Errorf("writing agent message to S3: %w", err) + } + + // agent.Index can yield the same (task, kind) pair more than once (e.g. a + // receipt's Ran() re-yields its invocation). Dedup by primary key so the + // batched INSERT below doesn't trip "ON CONFLICT DO UPDATE command cannot + // affect row a second time". Duplicates within a single message carry the + // same identifier by construction, so last-wins is safe. + type indexKey struct{ task, kind string } + rows := make(map[indexKey]string) + for _, entry := range index { + if entry.Invocation != nil { + invRoot, err := ipldutil.ToCID(entry.Invocation.Invocation.Link()) + if err != nil { + return fmt.Errorf("converting invocation root link to CID: %w", err) + } + rows[indexKey{entry.Invocation.Task.String(), "in"}] = fmt.Sprintf("%s@%s", invRoot, msgRoot) + } + if entry.Receipt != nil { + rcptRoot, err := ipldutil.ToCID(entry.Receipt.Receipt.Root().Link()) + if err != nil { + return fmt.Errorf("converting receipt root link to CID: %w", err) + } + rows[indexKey{entry.Receipt.Task.String(), "out"}] = fmt.Sprintf("%s@%s", rcptRoot, msgRoot) + } + } + + if len(rows) == 0 { + return nil + } + + placeholders := make([]string, 0, len(rows)) + args := make([]any, 0, 3*len(rows)) + i := 0 + for k, identifier := range rows { + placeholders = append(placeholders, fmt.Sprintf("($%d, $%d, $%d)", 3*i+1, 3*i+2, 3*i+3)) + args = append(args, k.task, k.kind, identifier) + i++ + } + query := "INSERT INTO agent_index (task, kind, identifier) VALUES " + + strings.Join(placeholders, ", ") + + " ON CONFLICT (task, kind) DO UPDATE SET identifier = EXCLUDED.identifier" + if _, err := s.pool.Exec(ctx, query, args...); err != nil { + return fmt.Errorf("writing agent index: %w", err) + } + return nil +} + +func toMessagePath(msg cid.Cid) string { + return fmt.Sprintf("%s/%s", msg, msg) +} diff --git a/pkg/store/blob_registry/blob_registry_test.go b/pkg/store/blob_registry/blob_registry_test.go index 9c78eff..9cd74da 100644 --- a/pkg/store/blob_registry/blob_registry_test.go +++ b/pkg/store/blob_registry/blob_registry_test.go @@ -12,12 +12,15 @@ import ( blobregistry "github.com/storacha/sprue/pkg/store/blob_registry" blobregistryaws "github.com/storacha/sprue/pkg/store/blob_registry/aws" "github.com/storacha/sprue/pkg/store/blob_registry/memory" + blobregistrypostgres "github.com/storacha/sprue/pkg/store/blob_registry/postgres" "github.com/storacha/sprue/pkg/store/consumer" consumeraws "github.com/storacha/sprue/pkg/store/consumer/aws" memoryconsumer "github.com/storacha/sprue/pkg/store/consumer/memory" + consumerpostgres "github.com/storacha/sprue/pkg/store/consumer/postgres" "github.com/storacha/sprue/pkg/store/metrics" metricsaws "github.com/storacha/sprue/pkg/store/metrics/aws" memorymetrics "github.com/storacha/sprue/pkg/store/metrics/memory" + metricspostgres "github.com/storacha/sprue/pkg/store/metrics/postgres" spacediffaws "github.com/storacha/sprue/pkg/store/space_diff/aws" memoryspacediff "github.com/storacha/sprue/pkg/store/space_diff/memory" "github.com/stretchr/testify/require" @@ -26,11 +29,12 @@ import ( type StoreKind string const ( - Memory StoreKind = "memory" - AWS StoreKind = "aws" + Memory StoreKind = "memory" + AWS StoreKind = "aws" + Postgres StoreKind = "postgres" ) -var storeKinds = []StoreKind{Memory, AWS} +var storeKinds = []StoreKind{Memory, AWS, Postgres} // storeBundle groups the blob registry with the dependency stores that tests // need to set up state (e.g. adding consumers before registering blobs). @@ -57,10 +61,34 @@ func makeStores(t *testing.T, k StoreKind) storeBundle { } case AWS: return createAWSStores(t) + case Postgres: + return createPostgresStores(t) } panic("unknown store kind") } +func createPostgresStores(t *testing.T) storeBundle { + if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { + if !testutil.IsDockerAvailable(t) { + t.Fatalf("docker is expected in CI linux testing environments, but wasn't found") + } + } + if !testutil.IsDockerAvailable(t) { + t.SkipNow() + } + pool := testutil.CreatePostgres(t) + consumerStore := consumerpostgres.New(pool) + spaceMetrics := metricspostgres.NewSpaceStore(pool) + adminMetrics := metricspostgres.New(pool) + registry := blobregistrypostgres.New(pool, consumerStore) + return storeBundle{ + registry: registry, + consumers: consumerStore, + spaceMetrics: spaceMetrics, + adminMetrics: adminMetrics, + } +} + func createAWSStores(t *testing.T) storeBundle { // This test expects docker to be running in linux CI environments and fails if it's not if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { diff --git a/pkg/store/blob_registry/postgres/store.go b/pkg/store/blob_registry/postgres/store.go new file mode 100644 index 0000000..9303fe7 --- /dev/null +++ b/pkg/store/blob_registry/postgres/store.go @@ -0,0 +1,266 @@ +// Package postgres provides a PostgreSQL-backed implementation of +// blob_registry.Store. Register and Deregister coordinate writes to +// blob_registry, space_diff, and the metrics stores in a single transaction. +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/ipfs/go-cid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/multiformats/go-multihash" + captypes "github.com/storacha/go-libstoracha/capabilities/types" + "github.com/storacha/go-libstoracha/digestutil" + "github.com/storacha/go-ucanto/did" + "github.com/storacha/sprue/pkg/store" + blobregistry "github.com/storacha/sprue/pkg/store/blob_registry" + "github.com/storacha/sprue/pkg/store/consumer" + "github.com/storacha/sprue/pkg/store/metrics" + pgmetrics "github.com/storacha/sprue/pkg/store/metrics/postgres" + pgspacediff "github.com/storacha/sprue/pkg/store/space_diff/postgres" +) + +const ( + defaultListLimit = 1000 + uniqueViolation = "23505" +) + +type Store struct { + pool *pgxpool.Pool + consumerStore consumer.Store +} + +var _ blobregistry.Store = (*Store)(nil) + +// New returns a Postgres-backed blob registry store. The consumerStore is used +// to fetch subscriptions for space_diff writes; the metrics and space_diff +// writes flow through package-level helpers from the metrics/postgres and +// space_diff/postgres packages. +func New(pool *pgxpool.Pool, consumerStore consumer.Store) *Store { + return &Store{pool: pool, consumerStore: consumerStore} +} + +func (s *Store) Initialize(ctx context.Context) error { return nil } + +func (s *Store) Get(ctx context.Context, space did.DID, digest multihash.Multihash) (blobregistry.Record, error) { + row := s.pool.QueryRow(ctx, ` + SELECT space, digest, size, cause, inserted_at + FROM blob_registry + WHERE space = $1 AND digest = $2 + `, space.String(), digestutil.Format(digest)) + rec, err := scanRecord(row) + if errors.Is(err, pgx.ErrNoRows) { + return blobregistry.Record{}, blobregistry.ErrEntryNotFound + } + if err != nil { + return blobregistry.Record{}, fmt.Errorf("getting blob registry entry: %w", err) + } + return rec, nil +} + +func (s *Store) Register(ctx context.Context, space did.DID, blob captypes.Blob, cause cid.Cid) error { + consumers, err := s.collectConsumers(ctx, space) + if err != nil { + return err + } + + tx, err := s.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("beginning transaction: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + if _, err := tx.Exec(ctx, ` + INSERT INTO blob_registry (space, digest, size, cause, inserted_at) + VALUES ($1, $2, $3, $4, $5) + `, space.String(), digestutil.Format(blob.Digest), int64(blob.Size), cause.String(), time.Now().UTC()); err != nil { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == uniqueViolation { + return blobregistry.ErrEntryExists + } + return fmt.Errorf("inserting blob registry entry: %w", err) + } + + receiptAt := time.Now() + for _, c := range consumers { + if err := pgspacediff.PutWith(ctx, tx, c.Provider, space, c.Subscription, cause, int64(blob.Size), receiptAt); err != nil { + return err + } + } + + inc := map[string]uint64{ + metrics.BlobAddTotalMetric: 1, + metrics.BlobAddSizeTotalMetric: blob.Size, + } + if err := pgmetrics.IncrementSpaceWith(ctx, tx, space, inc); err != nil { + return err + } + if err := pgmetrics.IncrementAdminWith(ctx, tx, inc); err != nil { + return err + } + + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("committing blob register: %w", err) + } + return nil +} + +func (s *Store) Deregister(ctx context.Context, space did.DID, digest multihash.Multihash, cause cid.Cid) error { + existing, err := s.Get(ctx, space, digest) + if err != nil { + return err + } + + consumers, err := s.collectConsumers(ctx, space) + if err != nil { + return err + } + + tx, err := s.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("beginning transaction: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + tag, err := tx.Exec(ctx, ` + DELETE FROM blob_registry WHERE space = $1 AND digest = $2 + `, space.String(), digestutil.Format(digest)) + if err != nil { + return fmt.Errorf("deleting blob registry entry: %w", err) + } + if tag.RowsAffected() == 0 { + return blobregistry.ErrEntryNotFound + } + + receiptAt := time.Now() + for _, c := range consumers { + if err := pgspacediff.PutWith(ctx, tx, c.Provider, space, c.Subscription, cause, -int64(existing.Blob.Size), receiptAt); err != nil { + return err + } + } + + inc := map[string]uint64{ + metrics.BlobRemoveTotalMetric: 1, + metrics.BlobRemoveSizeTotalMetric: existing.Blob.Size, + } + if err := pgmetrics.IncrementSpaceWith(ctx, tx, space, inc); err != nil { + return err + } + if err := pgmetrics.IncrementAdminWith(ctx, tx, inc); err != nil { + return err + } + + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("committing blob deregister: %w", err) + } + return nil +} + +func (s *Store) List(ctx context.Context, space did.DID, options ...blobregistry.ListOption) (store.Page[blobregistry.Record], error) { + cfg := blobregistry.ListConfig{} + for _, o := range options { + o(&cfg) + } + limit := defaultListLimit + if cfg.Limit != nil && *cfg.Limit > 0 { + limit = *cfg.Limit + } + + args := []any{space.String(), limit + 1} + query := ` + SELECT space, digest, size, cause, inserted_at + FROM blob_registry + WHERE space = $1 + ` + if cfg.Cursor != nil { + args = append(args, *cfg.Cursor) + query += ` AND digest > $3` + } + query += ` ORDER BY digest ASC LIMIT $2` + + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return store.Page[blobregistry.Record]{}, fmt.Errorf("listing blob registry entries: %w", err) + } + defer rows.Close() + + records := make([]blobregistry.Record, 0, limit) + for rows.Next() { + rec, err := scanRecord(rows) + if err != nil { + return store.Page[blobregistry.Record]{}, err + } + records = append(records, rec) + } + if err := rows.Err(); err != nil { + return store.Page[blobregistry.Record]{}, fmt.Errorf("iterating blob registry entries: %w", err) + } + + var cursor *string + if len(records) > limit { + last := digestutil.Format(records[limit-1].Blob.Digest) + cursor = &last + records = records[:limit] + } + return store.Page[blobregistry.Record]{Results: records, Cursor: cursor}, nil +} + +func (s *Store) collectConsumers(ctx context.Context, space did.DID) ([]consumer.Record, error) { + results, err := store.Collect(ctx, func(ctx context.Context, options store.PaginationConfig) (store.Page[consumer.Record], error) { + opts := []consumer.ListOption{} + if options.Cursor != nil { + opts = append(opts, consumer.WithListCursor(*options.Cursor)) + } + return s.consumerStore.List(ctx, space, opts...) + }) + if err != nil { + return nil, fmt.Errorf("listing consumers: %w", err) + } + if len(results) == 0 { + return nil, consumer.ErrConsumerNotFound + } + return results, nil +} + +type rowScanner interface { + Scan(dest ...any) error +} + +func scanRecord(row rowScanner) (blobregistry.Record, error) { + var ( + spaceStr string + digestStr string + size int64 + causeStr string + insertedAt time.Time + ) + if err := row.Scan(&spaceStr, &digestStr, &size, &causeStr, &insertedAt); err != nil { + return blobregistry.Record{}, err + } + space, err := did.Parse(spaceStr) + if err != nil { + return blobregistry.Record{}, fmt.Errorf("parsing space DID: %w", err) + } + digest, err := digestutil.Parse(digestStr) + if err != nil { + return blobregistry.Record{}, fmt.Errorf("parsing digest: %w", err) + } + cause, err := cid.Parse(causeStr) + if err != nil { + return blobregistry.Record{}, fmt.Errorf("parsing cause CID: %w", err) + } + return blobregistry.Record{ + Space: space, + Blob: captypes.Blob{ + Digest: digest, + Size: uint64(size), + }, + Cause: cause, + InsertedAt: insertedAt, + }, nil +} diff --git a/pkg/store/consumer/consumer_test.go b/pkg/store/consumer/consumer_test.go index 72a0f0d..0082f8a 100644 --- a/pkg/store/consumer/consumer_test.go +++ b/pkg/store/consumer/consumer_test.go @@ -11,17 +11,19 @@ import ( "github.com/storacha/sprue/pkg/store/consumer" consumeraws "github.com/storacha/sprue/pkg/store/consumer/aws" "github.com/storacha/sprue/pkg/store/consumer/memory" + consumerpostgres "github.com/storacha/sprue/pkg/store/consumer/postgres" "github.com/stretchr/testify/require" ) type StoreKind string const ( - Memory StoreKind = "memory" - AWS StoreKind = "aws" + Memory StoreKind = "memory" + AWS StoreKind = "aws" + Postgres StoreKind = "postgres" ) -var storeKinds = []StoreKind{Memory, AWS} +var storeKinds = []StoreKind{Memory, AWS, Postgres} func makeStore(t *testing.T, k StoreKind) consumer.Store { switch k { @@ -29,10 +31,25 @@ func makeStore(t *testing.T, k StoreKind) consumer.Store { return memory.New() case AWS: return createAWSStore(t) + case Postgres: + return createPostgresStore(t) } panic("unknown store kind") } +func createPostgresStore(t *testing.T) consumer.Store { + if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { + if !testutil.IsDockerAvailable(t) { + t.Fatalf("docker is expected in CI linux testing environments, but wasn't found") + } + } + if !testutil.IsDockerAvailable(t) { + t.SkipNow() + } + pool := testutil.CreatePostgres(t) + return consumerpostgres.New(pool) +} + func createAWSStore(t *testing.T) *consumeraws.Store { // This test expects docker to be running in linux CI environments and fails if it's not if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { diff --git a/pkg/store/consumer/postgres/store.go b/pkg/store/consumer/postgres/store.go new file mode 100644 index 0000000..177ce31 --- /dev/null +++ b/pkg/store/consumer/postgres/store.go @@ -0,0 +1,195 @@ +// Package postgres provides a PostgreSQL-backed implementation of consumer.Store. +package postgres + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/ipfs/go-cid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/storacha/go-ucanto/did" + "github.com/storacha/sprue/pkg/store" + "github.com/storacha/sprue/pkg/store/consumer" +) + +const ( + defaultListLimit = 1000 + uniqueViolation = "23505" +) + +type Store struct { + pool *pgxpool.Pool +} + +var _ consumer.Store = (*Store)(nil) + +func New(pool *pgxpool.Pool) *Store { + return &Store{pool: pool} +} + +func (s *Store) Initialize(ctx context.Context) error { return nil } + +func (s *Store) Add(ctx context.Context, provider did.DID, space did.DID, customer did.DID, subscription string, cause cid.Cid) error { + _, err := s.pool.Exec(ctx, ` + INSERT INTO consumer (subscription, provider, consumer, customer, cause, inserted_at) + VALUES ($1, $2, $3, $4, $5, $6) + `, subscription, provider.String(), space.String(), customer.String(), cause.String(), time.Now().UTC()) + if err != nil { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == uniqueViolation { + return consumer.ErrConsumerExists + } + return fmt.Errorf("adding consumer: %w", err) + } + return nil +} + +func (s *Store) Get(ctx context.Context, provider did.DID, space did.DID) (consumer.Record, error) { + row := s.pool.QueryRow(ctx, ` + SELECT subscription, provider, consumer, customer, cause + FROM consumer + WHERE consumer = $1 AND provider = $2 + LIMIT 1 + `, space.String(), provider.String()) + rec, err := scanRecord(row) + if errors.Is(err, pgx.ErrNoRows) { + return consumer.Record{}, consumer.ErrConsumerNotFound + } + if err != nil { + return consumer.Record{}, fmt.Errorf("getting consumer: %w", err) + } + return rec, nil +} + +func (s *Store) GetBySubscription(ctx context.Context, provider did.DID, subscription string) (consumer.Record, error) { + row := s.pool.QueryRow(ctx, ` + SELECT subscription, provider, consumer, customer, cause + FROM consumer + WHERE subscription = $1 AND provider = $2 + `, subscription, provider.String()) + rec, err := scanRecord(row) + if errors.Is(err, pgx.ErrNoRows) { + return consumer.Record{}, consumer.ErrConsumerNotFound + } + if err != nil { + return consumer.Record{}, fmt.Errorf("getting consumer by subscription: %w", err) + } + return rec, nil +} + +func (s *Store) List(ctx context.Context, space did.DID, options ...consumer.ListOption) (store.Page[consumer.Record], error) { + cfg := consumer.ListConfig{} + for _, opt := range options { + opt(&cfg) + } + return s.listByColumn(ctx, "consumer", space.String(), cfg.Cursor, cfg.Limit) +} + +func (s *Store) ListByCustomer(ctx context.Context, customer did.DID, options ...consumer.ListByCustomerOption) (store.Page[consumer.Record], error) { + cfg := consumer.ListByCustomerConfig{} + for _, opt := range options { + opt(&cfg) + } + return s.listByColumn(ctx, "customer", customer.String(), cfg.Cursor, cfg.Limit) +} + +func (s *Store) listByColumn(ctx context.Context, column, value string, cursor *string, limitPtr *int) (store.Page[consumer.Record], error) { + limit := defaultListLimit + if limitPtr != nil && *limitPtr > 0 { + limit = *limitPtr + } + + args := []any{value, limit + 1} + query := fmt.Sprintf(` + SELECT subscription, provider, consumer, customer, cause + FROM consumer + WHERE %s = $1 + `, column) + if cursor != nil { + provider, subscription, err := splitCursor(*cursor) + if err != nil { + return store.Page[consumer.Record]{}, fmt.Errorf("invalid cursor: %w", err) + } + args = append(args, provider, subscription) + query += ` AND (provider, subscription) > ($3, $4)` + } + query += ` ORDER BY provider, subscription LIMIT $2` + + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return store.Page[consumer.Record]{}, fmt.Errorf("listing consumers: %w", err) + } + defer rows.Close() + + records := make([]consumer.Record, 0, limit) + for rows.Next() { + rec, err := scanRecord(rows) + if err != nil { + return store.Page[consumer.Record]{}, err + } + records = append(records, rec) + } + if err := rows.Err(); err != nil { + return store.Page[consumer.Record]{}, fmt.Errorf("listing consumers: %w", err) + } + + var cursorOut *string + if len(records) > limit { + last := records[limit-1] + c := joinCursor(last.Provider.String(), last.Subscription) + cursorOut = &c + records = records[:limit] + } + return store.Page[consumer.Record]{Results: records, Cursor: cursorOut}, nil +} + +func joinCursor(provider, subscription string) string { + return provider + "|" + subscription +} + +func splitCursor(cursor string) (provider, subscription string, err error) { + parts := strings.SplitN(cursor, "|", 2) + if len(parts) != 2 { + return "", "", fmt.Errorf("malformed cursor %q", cursor) + } + return parts[0], parts[1], nil +} + +type rowScanner interface { + Scan(dest ...any) error +} + +func scanRecord(row rowScanner) (consumer.Record, error) { + var subscription, providerStr, consumerStr, customerStr, causeStr string + if err := row.Scan(&subscription, &providerStr, &consumerStr, &customerStr, &causeStr); err != nil { + return consumer.Record{}, err + } + provider, err := did.Parse(providerStr) + if err != nil { + return consumer.Record{}, fmt.Errorf("parsing provider DID: %w", err) + } + space, err := did.Parse(consumerStr) + if err != nil { + return consumer.Record{}, fmt.Errorf("parsing consumer DID: %w", err) + } + customerDID, err := did.Parse(customerStr) + if err != nil { + return consumer.Record{}, fmt.Errorf("parsing customer DID: %w", err) + } + cause, err := cid.Parse(causeStr) + if err != nil { + return consumer.Record{}, fmt.Errorf("parsing cause CID: %w", err) + } + return consumer.Record{ + Subscription: subscription, + Provider: provider, + Consumer: space, + Customer: customerDID, + Cause: cause, + }, nil +} diff --git a/pkg/store/customer/customer_test.go b/pkg/store/customer/customer_test.go index bf2c6eb..40e9efc 100644 --- a/pkg/store/customer/customer_test.go +++ b/pkg/store/customer/customer_test.go @@ -11,17 +11,19 @@ import ( "github.com/storacha/sprue/pkg/store/customer" customeraws "github.com/storacha/sprue/pkg/store/customer/aws" "github.com/storacha/sprue/pkg/store/customer/memory" + customerpostgres "github.com/storacha/sprue/pkg/store/customer/postgres" "github.com/stretchr/testify/require" ) type StoreKind string const ( - Memory StoreKind = "memory" - AWS StoreKind = "aws" + Memory StoreKind = "memory" + AWS StoreKind = "aws" + Postgres StoreKind = "postgres" ) -var storeKinds = []StoreKind{Memory, AWS} +var storeKinds = []StoreKind{Memory, AWS, Postgres} func makeStore(t *testing.T, k StoreKind) customer.Store { switch k { @@ -29,10 +31,25 @@ func makeStore(t *testing.T, k StoreKind) customer.Store { return memory.New() case AWS: return createAWSStore(t) + case Postgres: + return createPostgresStore(t) } panic("unknown store kind") } +func createPostgresStore(t *testing.T) customer.Store { + if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { + if !testutil.IsDockerAvailable(t) { + t.Fatalf("docker is expected in CI linux testing environments, but wasn't found") + } + } + if !testutil.IsDockerAvailable(t) { + t.SkipNow() + } + pool := testutil.CreatePostgres(t) + return customerpostgres.New(pool) +} + func createAWSStore(t *testing.T) customer.Store { // This test expects docker to be running in linux CI environments and fails if it's not if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { diff --git a/pkg/store/customer/postgres/store.go b/pkg/store/customer/postgres/store.go new file mode 100644 index 0000000..8528b41 --- /dev/null +++ b/pkg/store/customer/postgres/store.go @@ -0,0 +1,199 @@ +// Package postgres provides a PostgreSQL-backed implementation of customer.Store. +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/storacha/go-ucanto/did" + "github.com/storacha/sprue/pkg/store" + "github.com/storacha/sprue/pkg/store/customer" +) + +const ( + defaultListLimit = 1000 + uniqueViolation = "23505" +) + +// Store persists customer records in PostgreSQL. +type Store struct { + pool *pgxpool.Pool +} + +var _ customer.Store = (*Store)(nil) + +// New returns a Postgres-backed customer store. +func New(pool *pgxpool.Pool) *Store { + return &Store{pool: pool} +} + +// Initialize is a no-op. Schema is managed by the shared goose migrations. +func (s *Store) Initialize(ctx context.Context) error { return nil } + +func (s *Store) Get(ctx context.Context, customerID did.DID) (customer.Record, error) { + row := s.pool.QueryRow(ctx, ` + SELECT customer, account, product, details, reserved_capacity, inserted_at, updated_at + FROM customer + WHERE customer = $1 + `, customerID.String()) + rec, err := scanRecord(row) + if errors.Is(err, pgx.ErrNoRows) { + return customer.Record{}, customer.ErrCustomerNotFound + } + if err != nil { + return customer.Record{}, fmt.Errorf("getting customer: %w", err) + } + return rec, nil +} + +func (s *Store) Add(ctx context.Context, customerID did.DID, account *string, product did.DID, details map[string]any, reservedCapacity *uint64) error { + var detailsJSON []byte + if len(details) > 0 { + b, err := json.Marshal(details) + if err != nil { + return fmt.Errorf("marshalling details: %w", err) + } + detailsJSON = b + } + + var capacity *int64 + if reservedCapacity != nil { + c := int64(*reservedCapacity) + capacity = &c + } + + _, err := s.pool.Exec(ctx, ` + INSERT INTO customer (customer, account, product, details, reserved_capacity, inserted_at) + VALUES ($1, $2, $3, $4, $5, $6) + `, customerID.String(), account, product.String(), detailsJSON, capacity, time.Now().UTC()) + + if err != nil { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == uniqueViolation { + return customer.ErrCustomerExists + } + return fmt.Errorf("adding customer: %w", err) + } + return nil +} + +func (s *Store) List(ctx context.Context, options ...customer.ListOption) (store.Page[customer.Record], error) { + cfg := customer.ListConfig{} + for _, opt := range options { + opt(&cfg) + } + + limit := defaultListLimit + if cfg.Limit != nil && *cfg.Limit > 0 { + limit = *cfg.Limit + } + + args := []any{limit + 1} + query := ` + SELECT customer, account, product, details, reserved_capacity, inserted_at, updated_at + FROM customer + ` + if cfg.Cursor != nil { + args = append(args, *cfg.Cursor) + query += ` WHERE customer > $2` + } + query += ` ORDER BY customer ASC LIMIT $1` + + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return store.Page[customer.Record]{}, fmt.Errorf("listing customers: %w", err) + } + defer rows.Close() + + records := make([]customer.Record, 0, limit) + for rows.Next() { + rec, err := scanRecord(rows) + if err != nil { + return store.Page[customer.Record]{}, err + } + records = append(records, rec) + } + if err := rows.Err(); err != nil { + return store.Page[customer.Record]{}, fmt.Errorf("listing customers: %w", err) + } + + var cursor *string + if len(records) > limit { + last := records[limit-1].Customer.String() + cursor = &last + records = records[:limit] + } + return store.Page[customer.Record]{Results: records, Cursor: cursor}, nil +} + +func (s *Store) UpdateProduct(ctx context.Context, customerID did.DID, product did.DID) error { + tag, err := s.pool.Exec(ctx, ` + UPDATE customer + SET product = $1, updated_at = $2 + WHERE customer = $3 + `, product.String(), time.Now().UTC(), customerID.String()) + if err != nil { + return fmt.Errorf("updating customer product: %w", err) + } + if tag.RowsAffected() == 0 { + return customer.ErrCustomerNotFound + } + return nil +} + +// rowScanner abstracts pgx.Row and pgx.Rows for scanRecord. +type rowScanner interface { + Scan(dest ...any) error +} + +func scanRecord(row rowScanner) (customer.Record, error) { + var ( + customerStr string + account *string + productStr string + detailsJSON []byte + capacity *int64 + insertedAt time.Time + updatedAtRaw *time.Time + ) + if err := row.Scan(&customerStr, &account, &productStr, &detailsJSON, &capacity, &insertedAt, &updatedAtRaw); err != nil { + return customer.Record{}, err + } + + customerID, err := did.Parse(customerStr) + if err != nil { + return customer.Record{}, fmt.Errorf("parsing customer DID: %w", err) + } + product, err := did.Parse(productStr) + if err != nil { + return customer.Record{}, fmt.Errorf("parsing product DID: %w", err) + } + + rec := customer.Record{ + Customer: customerID, + Account: account, + Product: product, + InsertedAt: insertedAt, + } + if updatedAtRaw != nil { + rec.UpdatedAt = *updatedAtRaw + } + if capacity != nil { + c := uint64(*capacity) + rec.ReservedCapacity = &c + } + if len(detailsJSON) > 0 { + var details map[string]any + if err := json.Unmarshal(detailsJSON, &details); err != nil { + return customer.Record{}, fmt.Errorf("unmarshalling details: %w", err) + } + rec.Details = details + } + return rec, nil +} diff --git a/pkg/store/delegation/delegation_test.go b/pkg/store/delegation/delegation_test.go index 5dd04eb..a6bf1d5 100644 --- a/pkg/store/delegation/delegation_test.go +++ b/pkg/store/delegation/delegation_test.go @@ -13,17 +13,19 @@ import ( dlgstore "github.com/storacha/sprue/pkg/store/delegation" delegationaws "github.com/storacha/sprue/pkg/store/delegation/aws" "github.com/storacha/sprue/pkg/store/delegation/memory" + delegationpostgres "github.com/storacha/sprue/pkg/store/delegation/postgres" "github.com/stretchr/testify/require" ) type StoreKind string const ( - Memory StoreKind = "memory" - AWS StoreKind = "aws" + Memory StoreKind = "memory" + AWS StoreKind = "aws" + Postgres StoreKind = "postgres" ) -var storeKinds = []StoreKind{Memory, AWS} +var storeKinds = []StoreKind{Memory, AWS, Postgres} func makeStore(t *testing.T, k StoreKind) dlgstore.Store { switch k { @@ -31,10 +33,30 @@ func makeStore(t *testing.T, k StoreKind) dlgstore.Store { return memory.New() case AWS: return createAWSStore(t) + case Postgres: + return createPostgresStore(t) } panic("unknown store kind") } +func createPostgresStore(t *testing.T) dlgstore.Store { + if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { + if !testutil.IsDockerAvailable(t) { + t.Fatalf("docker is expected in CI linux testing environments, but wasn't found") + } + } + if !testutil.IsDockerAvailable(t) { + t.SkipNow() + } + pool := testutil.CreatePostgres(t) + s3Endpoint := testutil.CreateS3(t) + s3Client := testutil.NewS3Client(t, s3Endpoint) + + s := delegationpostgres.New(pool, s3Client, "delegation-"+uuid.NewString()) + require.NoError(t, s.Initialize(t.Context())) + return s +} + func createAWSStore(t *testing.T) dlgstore.Store { // This test expects docker to be running in linux CI environments and fails if it's not if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { diff --git a/pkg/store/delegation/postgres/store.go b/pkg/store/delegation/postgres/store.go new file mode 100644 index 0000000..08ef68e --- /dev/null +++ b/pkg/store/delegation/postgres/store.go @@ -0,0 +1,168 @@ +// Package postgres provides a PostgreSQL-backed implementation of delegation.Store. +// Metadata lives in Postgres; the delegation payload archives remain in S3. +package postgres + +import ( + "bytes" + "context" + "fmt" + "io" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/ipfs/go-cid" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/storacha/go-ucanto/core/delegation" + "github.com/storacha/go-ucanto/did" + "github.com/storacha/sprue/pkg/store" + dlgstore "github.com/storacha/sprue/pkg/store/delegation" +) + +const defaultListLimit = 1000 + +type Store struct { + pool *pgxpool.Pool + s3 *s3.Client + bucketName string +} + +var _ dlgstore.Store = (*Store)(nil) + +func New(pool *pgxpool.Pool, s3Client *s3.Client, bucketName string) *Store { + return &Store{pool: pool, s3: s3Client, bucketName: bucketName} +} + +// Initialize ensures the S3 bucket exists. Table schema is managed by goose. +func (s *Store) Initialize(ctx context.Context) error { + if _, err := s.s3.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &s.bucketName}); err != nil { + if _, err := s.s3.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: &s.bucketName}); err != nil { + return fmt.Errorf("creating S3 bucket %q: %w", s.bucketName, err) + } + } + return nil +} + +func (s *Store) PutMany(ctx context.Context, delegations []delegation.Delegation, cause cid.Cid) error { + now := time.Now().UTC() + for _, dlg := range delegations { + link := dlg.Root().Link().String() + + body, err := io.ReadAll(dlg.Archive()) + if err != nil { + return fmt.Errorf("archiving delegation %s: %w", link, err) + } + if _, err := s.s3.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &s.bucketName, + Key: aws.String(link), + Body: bytes.NewReader(body), + }); err != nil { + return fmt.Errorf("storing delegation %s in S3: %w", link, err) + } + + var causeStr *string + if cause != cid.Undef { + c := cause.String() + causeStr = &c + } + var expiration *int64 + if exp := dlg.Expiration(); exp != nil { + e := int64(*exp) + expiration = &e + } + + if _, err := s.pool.Exec(ctx, ` + INSERT INTO delegation (link, audience, issuer, cause, expiration, inserted_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $6) + ON CONFLICT (link) DO UPDATE + SET audience = EXCLUDED.audience, + issuer = EXCLUDED.issuer, + cause = EXCLUDED.cause, + expiration = EXCLUDED.expiration, + updated_at = EXCLUDED.updated_at + `, link, dlg.Audience().DID().String(), dlg.Issuer().DID().String(), causeStr, expiration, now); err != nil { + return fmt.Errorf("indexing delegation %s: %w", link, err) + } + } + return nil +} + +func (s *Store) ListByAudience(ctx context.Context, audience did.DID, options ...dlgstore.ListByAudienceOption) (store.Page[delegation.Delegation], error) { + cfg := dlgstore.ListByAudienceConfig{} + for _, opt := range options { + opt(&cfg) + } + limit := defaultListLimit + if cfg.Limit != nil && *cfg.Limit > 0 { + limit = *cfg.Limit + } + + args := []any{audience.String(), limit + 1} + query := ` + SELECT link + FROM delegation + WHERE audience = $1 + ` + if cfg.Cursor != nil { + args = append(args, *cfg.Cursor) + query += ` AND link > $3` + } + query += ` ORDER BY link ASC LIMIT $2` + + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return store.Page[delegation.Delegation]{}, fmt.Errorf("querying delegations by audience: %w", err) + } + defer rows.Close() + + links := make([]string, 0, limit) + for rows.Next() { + var link string + if err := rows.Scan(&link); err != nil { + return store.Page[delegation.Delegation]{}, fmt.Errorf("scanning delegation: %w", err) + } + links = append(links, link) + } + if err := rows.Err(); err != nil { + return store.Page[delegation.Delegation]{}, fmt.Errorf("iterating delegations: %w", err) + } + + var cursor *string + if len(links) > limit { + last := links[limit-1] + cursor = &last + links = links[:limit] + } + + results := make([]delegation.Delegation, 0, len(links)) + for _, link := range links { + dlg, err := s.fetchDelegation(ctx, link) + if err != nil { + return store.Page[delegation.Delegation]{}, fmt.Errorf("fetching delegation %s: %w", link, err) + } + results = append(results, dlg) + } + + return store.Page[delegation.Delegation]{Results: results, Cursor: cursor}, nil +} + +func (s *Store) fetchDelegation(ctx context.Context, link string) (delegation.Delegation, error) { + out, err := s.s3.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &s.bucketName, + Key: aws.String(link), + }) + if err != nil { + return nil, fmt.Errorf("getting delegation from S3: %w", err) + } + defer out.Body.Close() + + data, err := io.ReadAll(out.Body) + if err != nil { + return nil, fmt.Errorf("reading delegation from S3: %w", err) + } + dlg, err := delegation.Extract(data) + if err != nil { + return nil, fmt.Errorf("extracting delegation: %w", err) + } + return dlg, nil +} diff --git a/pkg/store/metrics/metrics_test.go b/pkg/store/metrics/metrics_test.go index b7b3769..a75ecbf 100644 --- a/pkg/store/metrics/metrics_test.go +++ b/pkg/store/metrics/metrics_test.go @@ -9,17 +9,19 @@ import ( "github.com/storacha/sprue/pkg/store/metrics" metricsaws "github.com/storacha/sprue/pkg/store/metrics/aws" "github.com/storacha/sprue/pkg/store/metrics/memory" + metricspostgres "github.com/storacha/sprue/pkg/store/metrics/postgres" "github.com/stretchr/testify/require" ) type StoreKind string const ( - Memory StoreKind = "memory" - AWS StoreKind = "aws" + Memory StoreKind = "memory" + AWS StoreKind = "aws" + Postgres StoreKind = "postgres" ) -var storeKinds = []StoreKind{Memory, AWS} +var storeKinds = []StoreKind{Memory, AWS, Postgres} func makeStore(t *testing.T, k StoreKind) metrics.Store { switch k { @@ -27,6 +29,8 @@ func makeStore(t *testing.T, k StoreKind) metrics.Store { return memory.New() case AWS: return createAWSStore(t) + case Postgres: + return createPostgresStore(t) } panic("unknown store kind") } @@ -37,10 +41,38 @@ func makeSpaceStore(t *testing.T, k StoreKind) metrics.SpaceStore { return memory.NewSpaceStore() case AWS: return createAWSSpaceStore(t) + case Postgres: + return createPostgresSpaceStore(t) } panic("unknown store kind") } +func createPostgresStore(t *testing.T) *metricspostgres.Store { + if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { + if !testutil.IsDockerAvailable(t) { + t.Fatalf("docker is expected in CI linux testing environments, but wasn't found") + } + } + if !testutil.IsDockerAvailable(t) { + t.SkipNow() + } + pool := testutil.CreatePostgres(t) + return metricspostgres.New(pool) +} + +func createPostgresSpaceStore(t *testing.T) *metricspostgres.SpaceStore { + if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { + if !testutil.IsDockerAvailable(t) { + t.Fatalf("docker is expected in CI linux testing environments, but wasn't found") + } + } + if !testutil.IsDockerAvailable(t) { + t.SkipNow() + } + pool := testutil.CreatePostgres(t) + return metricspostgres.NewSpaceStore(pool) +} + func createAWSStore(t *testing.T) *metricsaws.Store { // This test expects docker to be running in linux CI environments and fails if it's not if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { diff --git a/pkg/store/metrics/postgres/store.go b/pkg/store/metrics/postgres/store.go new file mode 100644 index 0000000..721586b --- /dev/null +++ b/pkg/store/metrics/postgres/store.go @@ -0,0 +1,130 @@ +// Package postgres provides PostgreSQL-backed implementations of metrics.Store +// and metrics.SpaceStore. +package postgres + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/storacha/go-ucanto/did" + "github.com/storacha/sprue/pkg/store/metrics" +) + +// pgxExec is the common Exec surface shared by *pgxpool.Pool and pgx.Tx. +type pgxExec interface { + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) +} + +// Store persists global admin metrics in PostgreSQL. +type Store struct { + pool *pgxpool.Pool +} + +var _ metrics.Store = (*Store)(nil) + +func New(pool *pgxpool.Pool) *Store { + return &Store{pool: pool} +} + +func (s *Store) Initialize(ctx context.Context) error { return nil } + +func (s *Store) Get(ctx context.Context) (map[string]uint64, error) { + rows, err := s.pool.Query(ctx, `SELECT name, value FROM admin_metrics`) + if err != nil { + return nil, fmt.Errorf("querying admin metrics: %w", err) + } + defer rows.Close() + result := map[string]uint64{} + for rows.Next() { + var name string + var value int64 + if err := rows.Scan(&name, &value); err != nil { + return nil, fmt.Errorf("scanning admin metrics: %w", err) + } + result[name] = uint64(value) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating admin metrics: %w", err) + } + return result, nil +} + +func (s *Store) IncrementTotals(ctx context.Context, inc map[string]uint64) error { + if len(inc) == 0 { + return nil + } + return IncrementAdminWith(ctx, s.pool, inc) +} + +// IncrementAdminWith increments the admin metrics via the provided querier, +// enabling inclusion in an external transaction. +func IncrementAdminWith(ctx context.Context, q pgxExec, inc map[string]uint64) error { + for metric, delta := range inc { + if _, err := q.Exec(ctx, ` + INSERT INTO admin_metrics (name, value) + VALUES ($1, $2) + ON CONFLICT (name) DO UPDATE SET value = admin_metrics.value + EXCLUDED.value + `, metric, int64(delta)); err != nil { + return fmt.Errorf("incrementing admin metric %q: %w", metric, err) + } + } + return nil +} + +// SpaceStore persists per-space metrics in PostgreSQL. +type SpaceStore struct { + pool *pgxpool.Pool +} + +var _ metrics.SpaceStore = (*SpaceStore)(nil) + +func NewSpaceStore(pool *pgxpool.Pool) *SpaceStore { + return &SpaceStore{pool: pool} +} + +func (s *SpaceStore) Initialize(ctx context.Context) error { return nil } + +func (s *SpaceStore) Get(ctx context.Context, space did.DID) (map[string]uint64, error) { + rows, err := s.pool.Query(ctx, `SELECT name, value FROM space_metrics WHERE space = $1`, space.String()) + if err != nil { + return nil, fmt.Errorf("querying space metrics: %w", err) + } + defer rows.Close() + result := map[string]uint64{} + for rows.Next() { + var name string + var value int64 + if err := rows.Scan(&name, &value); err != nil { + return nil, fmt.Errorf("scanning space metrics: %w", err) + } + result[name] = uint64(value) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating space metrics: %w", err) + } + return result, nil +} + +func (s *SpaceStore) IncrementTotals(ctx context.Context, space did.DID, inc map[string]uint64) error { + if len(inc) == 0 { + return nil + } + return IncrementSpaceWith(ctx, s.pool, space, inc) +} + +// IncrementSpaceWith increments per-space metrics via the provided querier, +// enabling inclusion in an external transaction. +func IncrementSpaceWith(ctx context.Context, q pgxExec, space did.DID, inc map[string]uint64) error { + for metric, delta := range inc { + if _, err := q.Exec(ctx, ` + INSERT INTO space_metrics (space, name, value) + VALUES ($1, $2, $3) + ON CONFLICT (space, name) DO UPDATE SET value = space_metrics.value + EXCLUDED.value + `, space.String(), metric, int64(delta)); err != nil { + return fmt.Errorf("incrementing space metric %q: %w", metric, err) + } + } + return nil +} diff --git a/pkg/store/replica/postgres/store.go b/pkg/store/replica/postgres/store.go new file mode 100644 index 0000000..e4a2d8d --- /dev/null +++ b/pkg/store/replica/postgres/store.go @@ -0,0 +1,163 @@ +// Package postgres provides a PostgreSQL-backed implementation of replica.Store. +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/ipfs/go-cid" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/multiformats/go-multihash" + "github.com/storacha/go-libstoracha/digestutil" + "github.com/storacha/go-ucanto/did" + "github.com/storacha/sprue/pkg/store/replica" +) + +const uniqueViolation = "23505" + +type Store struct { + pool *pgxpool.Pool +} + +var _ replica.Store = (*Store)(nil) + +func New(pool *pgxpool.Pool) *Store { + return &Store{pool: pool} +} + +func (s *Store) Initialize(ctx context.Context) error { return nil } + +func (s *Store) Add(ctx context.Context, space did.DID, digest multihash.Multihash, provider did.DID, status replica.ReplicationStatus, cause cid.Cid) error { + now := time.Now().UTC() + _, err := s.pool.Exec(ctx, ` + INSERT INTO replica (space, digest, provider, status, cause, inserted_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $6) + `, space.String(), digestutil.Format(digest), provider.String(), status.String(), cause.String(), now) + if err != nil { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == uniqueViolation { + return replica.ErrReplicaExists + } + return fmt.Errorf("adding replica: %w", err) + } + return nil +} + +func (s *Store) List(ctx context.Context, space did.DID, digest multihash.Multihash) ([]replica.Record, error) { + rows, err := s.pool.Query(ctx, ` + SELECT space, digest, provider, status, cause, inserted_at, updated_at + FROM replica + WHERE space = $1 AND digest = $2 + ORDER BY provider ASC + `, space.String(), digestutil.Format(digest)) + if err != nil { + return nil, fmt.Errorf("listing replicas: %w", err) + } + defer rows.Close() + + var records []replica.Record + for rows.Next() { + rec, err := scanRecord(rows) + if err != nil { + return nil, err + } + records = append(records, rec) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating replicas: %w", err) + } + return records, nil +} + +func (s *Store) Retry(ctx context.Context, space did.DID, digest multihash.Multihash, provider did.DID, status replica.ReplicationStatus, cause cid.Cid) error { + tag, err := s.pool.Exec(ctx, ` + UPDATE replica + SET status = $1, cause = $2, updated_at = $3 + WHERE space = $4 AND digest = $5 AND provider = $6 + `, status.String(), cause.String(), time.Now().UTC(), space.String(), digestutil.Format(digest), provider.String()) + if err != nil { + return fmt.Errorf("retrying replica: %w", err) + } + if tag.RowsAffected() == 0 { + return replica.ErrReplicaNotFound + } + return nil +} + +func (s *Store) SetStatus(ctx context.Context, space did.DID, digest multihash.Multihash, provider did.DID, status replica.ReplicationStatus) error { + tag, err := s.pool.Exec(ctx, ` + UPDATE replica + SET status = $1, updated_at = $2 + WHERE space = $3 AND digest = $4 AND provider = $5 + `, status.String(), time.Now().UTC(), space.String(), digestutil.Format(digest), provider.String()) + if err != nil { + return fmt.Errorf("setting replica status: %w", err) + } + if tag.RowsAffected() == 0 { + return replica.ErrReplicaNotFound + } + return nil +} + +type rowScanner interface { + Scan(dest ...any) error +} + +func scanRecord(row rowScanner) (replica.Record, error) { + var ( + spaceStr string + digestStr string + providerStr string + statusStr string + causeStr string + insertedAt time.Time + updatedAt time.Time + ) + if err := row.Scan(&spaceStr, &digestStr, &providerStr, &statusStr, &causeStr, &insertedAt, &updatedAt); err != nil { + return replica.Record{}, err + } + space, err := did.Parse(spaceStr) + if err != nil { + return replica.Record{}, fmt.Errorf("parsing space DID: %w", err) + } + digest, err := digestutil.Parse(digestStr) + if err != nil { + return replica.Record{}, fmt.Errorf("parsing digest: %w", err) + } + provider, err := did.Parse(providerStr) + if err != nil { + return replica.Record{}, fmt.Errorf("parsing provider DID: %w", err) + } + status, err := parseStatus(statusStr) + if err != nil { + return replica.Record{}, err + } + cause, err := cid.Parse(causeStr) + if err != nil { + return replica.Record{}, fmt.Errorf("parsing cause CID: %w", err) + } + return replica.Record{ + Space: space, + Digest: digest, + Provider: provider, + Status: status, + Cause: cause, + CreatedAt: insertedAt, + UpdatedAt: updatedAt, + }, nil +} + +func parseStatus(s string) (replica.ReplicationStatus, error) { + switch s { + case replica.Allocated.String(): + return replica.Allocated, nil + case replica.Transferred.String(): + return replica.Transferred, nil + case replica.Failed.String(): + return replica.Failed, nil + } + return 0, fmt.Errorf("unknown replication status %q", s) +} diff --git a/pkg/store/replica/replica_test.go b/pkg/store/replica/replica_test.go index 96fe023..94a2a76 100644 --- a/pkg/store/replica/replica_test.go +++ b/pkg/store/replica/replica_test.go @@ -9,17 +9,19 @@ import ( "github.com/storacha/sprue/pkg/store/replica" replicaaws "github.com/storacha/sprue/pkg/store/replica/aws" "github.com/storacha/sprue/pkg/store/replica/memory" + replicapostgres "github.com/storacha/sprue/pkg/store/replica/postgres" "github.com/stretchr/testify/require" ) type StoreKind string const ( - Memory StoreKind = "memory" - AWS StoreKind = "aws" + Memory StoreKind = "memory" + AWS StoreKind = "aws" + Postgres StoreKind = "postgres" ) -var storeKinds = []StoreKind{Memory, AWS} +var storeKinds = []StoreKind{Memory, AWS, Postgres} func makeStore(t *testing.T, k StoreKind) replica.Store { switch k { @@ -27,10 +29,25 @@ func makeStore(t *testing.T, k StoreKind) replica.Store { return memory.New() case AWS: return createAWSStore(t) + case Postgres: + return createPostgresStore(t) } panic("unknown store kind") } +func createPostgresStore(t *testing.T) replica.Store { + if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { + if !testutil.IsDockerAvailable(t) { + t.Fatalf("docker is expected in CI linux testing environments, but wasn't found") + } + } + if !testutil.IsDockerAvailable(t) { + t.SkipNow() + } + pool := testutil.CreatePostgres(t) + return replicapostgres.New(pool) +} + func createAWSStore(t *testing.T) replica.Store { // This test expects docker to be running in linux CI environments and fails if it's not if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { diff --git a/pkg/store/revocation/postgres/store.go b/pkg/store/revocation/postgres/store.go new file mode 100644 index 0000000..6a121ea --- /dev/null +++ b/pkg/store/revocation/postgres/store.go @@ -0,0 +1,113 @@ +// Package postgres provides a PostgreSQL-backed implementation of revocation.Store. +package postgres + +import ( + "context" + "fmt" + "time" + + "github.com/ipfs/go-cid" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/storacha/go-ucanto/did" + "github.com/storacha/sprue/pkg/store/revocation" +) + +type Store struct { + pool *pgxpool.Pool +} + +var _ revocation.Store = (*Store)(nil) + +func New(pool *pgxpool.Pool) *Store { + return &Store{pool: pool} +} + +func (s *Store) Initialize(ctx context.Context) error { return nil } + +// Add inserts a revocation if one does not already exist for the given +// (delegation, scope). Matches the AWS semantics of "leave existing unchanged". +func (s *Store) Add(ctx context.Context, delegation cid.Cid, scope did.DID, cause cid.Cid) error { + _, err := s.pool.Exec(ctx, ` + INSERT INTO revocation (revoke, scope, cause, inserted_at) + VALUES ($1, $2, $3, $4) + ON CONFLICT (revoke, scope) DO NOTHING + `, delegation.String(), scope.String(), cause.String(), time.Now().UTC()) + if err != nil { + return fmt.Errorf("adding revocation: %w", err) + } + return nil +} + +// Reset replaces all existing revocations for the given delegation with a +// single revocation for the given scope. +func (s *Store) Reset(ctx context.Context, delegation cid.Cid, scope did.DID, cause cid.Cid) error { + tx, err := s.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("beginning transaction: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + if _, err := tx.Exec(ctx, `DELETE FROM revocation WHERE revoke = $1`, delegation.String()); err != nil { + return fmt.Errorf("clearing revocation: %w", err) + } + if _, err := tx.Exec(ctx, ` + INSERT INTO revocation (revoke, scope, cause, inserted_at) + VALUES ($1, $2, $3, $4) + `, delegation.String(), scope.String(), cause.String(), time.Now().UTC()); err != nil { + return fmt.Errorf("inserting revocation: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("committing revocation reset: %w", err) + } + return nil +} + +func (s *Store) Find(ctx context.Context, delegations []cid.Cid) (map[cid.Cid]map[did.DID]cid.Cid, error) { + result := map[cid.Cid]map[did.DID]cid.Cid{} + if len(delegations) == 0 { + return result, nil + } + dlgStrs := make([]string, len(delegations)) + for i, d := range delegations { + dlgStrs[i] = d.String() + } + + rows, err := s.pool.Query(ctx, ` + SELECT revoke, scope, cause + FROM revocation + WHERE revoke = ANY($1) + `, dlgStrs) + if err != nil { + return nil, fmt.Errorf("finding revocations: %w", err) + } + defer rows.Close() + + for rows.Next() { + var revokeStr, scopeStr, causeStr string + if err := rows.Scan(&revokeStr, &scopeStr, &causeStr); err != nil { + return nil, fmt.Errorf("scanning revocation: %w", err) + } + dlgCID, err := cid.Parse(revokeStr) + if err != nil { + return nil, fmt.Errorf("parsing delegation CID: %w", err) + } + scopeDID, err := did.Parse(scopeStr) + if err != nil { + return nil, fmt.Errorf("parsing scope DID: %w", err) + } + cause, err := cid.Parse(causeStr) + if err != nil { + return nil, fmt.Errorf("parsing cause CID: %w", err) + } + scopes, ok := result[dlgCID] + if !ok { + scopes = map[did.DID]cid.Cid{} + result[dlgCID] = scopes + } + scopes[scopeDID] = cause + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating revocations: %w", err) + } + return result, nil +} diff --git a/pkg/store/revocation/revocation_test.go b/pkg/store/revocation/revocation_test.go index 2345b68..6f45605 100644 --- a/pkg/store/revocation/revocation_test.go +++ b/pkg/store/revocation/revocation_test.go @@ -10,17 +10,19 @@ import ( "github.com/storacha/sprue/pkg/store/revocation" revocationaws "github.com/storacha/sprue/pkg/store/revocation/aws" "github.com/storacha/sprue/pkg/store/revocation/memory" + revocationpostgres "github.com/storacha/sprue/pkg/store/revocation/postgres" "github.com/stretchr/testify/require" ) type StoreKind string const ( - Memory StoreKind = "memory" - AWS StoreKind = "aws" + Memory StoreKind = "memory" + AWS StoreKind = "aws" + Postgres StoreKind = "postgres" ) -var storeKinds = []StoreKind{Memory, AWS} +var storeKinds = []StoreKind{Memory, AWS, Postgres} func makeStore(t *testing.T, k StoreKind) revocation.Store { switch k { @@ -28,10 +30,25 @@ func makeStore(t *testing.T, k StoreKind) revocation.Store { return memory.New() case AWS: return createAWSStore(t) + case Postgres: + return createPostgresStore(t) } panic("unknown store kind") } +func createPostgresStore(t *testing.T) revocation.Store { + if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { + if !testutil.IsDockerAvailable(t) { + t.Fatalf("docker is expected in CI linux testing environments, but wasn't found") + } + } + if !testutil.IsDockerAvailable(t) { + t.SkipNow() + } + pool := testutil.CreatePostgres(t) + return revocationpostgres.New(pool) +} + func createAWSStore(t *testing.T) revocation.Store { // This test expects docker to be running in linux CI environments and fails if it's not if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { diff --git a/pkg/store/space_diff/postgres/store.go b/pkg/store/space_diff/postgres/store.go new file mode 100644 index 0000000..5c83d7e --- /dev/null +++ b/pkg/store/space_diff/postgres/store.go @@ -0,0 +1,172 @@ +// Package postgres provides a PostgreSQL-backed implementation of space_diff.Store. +package postgres + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/ipfs/go-cid" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/storacha/go-ucanto/did" + "github.com/storacha/sprue/pkg/store" + spacediff "github.com/storacha/sprue/pkg/store/space_diff" +) + +const defaultListLimit = 1000 + +// pgxExec is the common Exec surface shared by *pgxpool.Pool and pgx.Tx. +type pgxExec interface { + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) +} + +type Store struct { + pool *pgxpool.Pool +} + +var _ spacediff.Store = (*Store)(nil) + +func New(pool *pgxpool.Pool) *Store { + return &Store{pool: pool} +} + +func (s *Store) Initialize(ctx context.Context) error { return nil } + +func (s *Store) Put(ctx context.Context, provider did.DID, space did.DID, subscription string, cause cid.Cid, delta int64, receiptAt time.Time) error { + return PutWith(ctx, s.pool, provider, space, subscription, cause, delta, receiptAt) +} + +// PutWith inserts a space diff row using the provided querier, allowing the +// write to participate in an external transaction. It exists so blob_registry +// can batch space-diff writes with its own updates in one atomic unit (matching +// the DynamoDB TransactWriteItems behaviour used by the AWS backend). +func PutWith(ctx context.Context, q pgxExec, provider did.DID, space did.DID, subscription string, cause cid.Cid, delta int64, receiptAt time.Time) error { + _, err := q.Exec(ctx, ` + INSERT INTO space_diff (provider, space, receipt_at, cause, subscription, delta, inserted_at) + VALUES ($1, $2, $3, $4, $5, $6, $7) + `, provider.String(), space.String(), receiptAt.UTC(), cause.String(), subscription, delta, time.Now().UTC()) + if err != nil { + return fmt.Errorf("putting space diff: %w", err) + } + return nil +} + +func (s *Store) List(ctx context.Context, provider did.DID, space did.DID, after time.Time, options ...spacediff.ListOption) (store.Page[spacediff.DifferenceRecord], error) { + cfg := spacediff.ListConfig{} + for _, opt := range options { + opt(&cfg) + } + limit := defaultListLimit + if cfg.Limit != nil && *cfg.Limit > 0 { + limit = *cfg.Limit + } + + var ( + conds []string + args []any + ) + args = append(args, provider.String(), space.String()) + conds = append(conds, "provider = $1", "space = $2") + + if cfg.Cursor != nil { + receiptAt, cause, err := decodeCursor(*cfg.Cursor) + if err != nil { + return store.Page[spacediff.DifferenceRecord]{}, fmt.Errorf("invalid cursor: %w", err) + } + args = append(args, receiptAt, cause) + conds = append(conds, fmt.Sprintf("(receipt_at, cause) > ($%d, $%d)", len(args)-1, len(args))) + } else if !after.IsZero() { + args = append(args, after.UTC()) + conds = append(conds, fmt.Sprintf("receipt_at > $%d", len(args))) + } + + args = append(args, limit+1) + query := fmt.Sprintf(` + SELECT provider, space, subscription, cause, delta, receipt_at, inserted_at + FROM space_diff + WHERE %s + ORDER BY receipt_at ASC, cause ASC + LIMIT $%d + `, strings.Join(conds, " AND "), len(args)) + + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return store.Page[spacediff.DifferenceRecord]{}, fmt.Errorf("listing space diffs: %w", err) + } + defer rows.Close() + + records := make([]spacediff.DifferenceRecord, 0, limit) + for rows.Next() { + var ( + providerStr string + spaceStr string + subscription string + causeStr string + delta int64 + receiptAt time.Time + insertedAt time.Time + ) + if err := rows.Scan(&providerStr, &spaceStr, &subscription, &causeStr, &delta, &receiptAt, &insertedAt); err != nil { + return store.Page[spacediff.DifferenceRecord]{}, fmt.Errorf("scanning space diff: %w", err) + } + providerDID, err := did.Parse(providerStr) + if err != nil { + return store.Page[spacediff.DifferenceRecord]{}, fmt.Errorf("parsing provider DID: %w", err) + } + spaceDID, err := did.Parse(spaceStr) + if err != nil { + return store.Page[spacediff.DifferenceRecord]{}, fmt.Errorf("parsing space DID: %w", err) + } + cause, err := cid.Parse(causeStr) + if err != nil { + return store.Page[spacediff.DifferenceRecord]{}, fmt.Errorf("parsing cause CID: %w", err) + } + records = append(records, spacediff.DifferenceRecord{ + Provider: providerDID, + Space: spaceDID, + Subscription: subscription, + Cause: cause, + Delta: delta, + ReceiptAt: receiptAt, + InsertedAt: insertedAt, + }) + } + if err := rows.Err(); err != nil { + return store.Page[spacediff.DifferenceRecord]{}, fmt.Errorf("iterating space diffs: %w", err) + } + + var cursor *string + if len(records) > limit { + last := records[limit-1] + c := encodeCursor(last.ReceiptAt, last.Cause.String()) + cursor = &c + records = records[:limit] + } + return store.Page[spacediff.DifferenceRecord]{Results: records, Cursor: cursor}, nil +} + +type cursorPayload struct { + ReceiptAt time.Time `json:"r"` + Cause string `json:"c"` +} + +func encodeCursor(receiptAt time.Time, cause string) string { + b, _ := json.Marshal(cursorPayload{ReceiptAt: receiptAt.UTC(), Cause: cause}) + return base64.RawURLEncoding.EncodeToString(b) +} + +func decodeCursor(cursor string) (time.Time, string, error) { + b, err := base64.RawURLEncoding.DecodeString(cursor) + if err != nil { + return time.Time{}, "", err + } + var p cursorPayload + if err := json.Unmarshal(b, &p); err != nil { + return time.Time{}, "", err + } + return p.ReceiptAt, p.Cause, nil +} diff --git a/pkg/store/space_diff/space_diff_test.go b/pkg/store/space_diff/space_diff_test.go index bc0e11b..9314cd4 100644 --- a/pkg/store/space_diff/space_diff_test.go +++ b/pkg/store/space_diff/space_diff_test.go @@ -12,17 +12,19 @@ import ( spacediff "github.com/storacha/sprue/pkg/store/space_diff" spacediffaws "github.com/storacha/sprue/pkg/store/space_diff/aws" "github.com/storacha/sprue/pkg/store/space_diff/memory" + spacediffpostgres "github.com/storacha/sprue/pkg/store/space_diff/postgres" "github.com/stretchr/testify/require" ) type StoreKind string const ( - Memory StoreKind = "memory" - AWS StoreKind = "aws" + Memory StoreKind = "memory" + AWS StoreKind = "aws" + Postgres StoreKind = "postgres" ) -var storeKinds = []StoreKind{Memory, AWS} +var storeKinds = []StoreKind{Memory, AWS, Postgres} func makeStore(t *testing.T, k StoreKind) spacediff.Store { switch k { @@ -30,10 +32,25 @@ func makeStore(t *testing.T, k StoreKind) spacediff.Store { return memory.New() case AWS: return createAWSStore(t) + case Postgres: + return createPostgresStore(t) } panic("unknown store kind") } +func createPostgresStore(t *testing.T) *spacediffpostgres.Store { + if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { + if !testutil.IsDockerAvailable(t) { + t.Fatalf("docker is expected in CI linux testing environments, but wasn't found") + } + } + if !testutil.IsDockerAvailable(t) { + t.SkipNow() + } + pool := testutil.CreatePostgres(t) + return spacediffpostgres.New(pool) +} + func createAWSStore(t *testing.T) *spacediffaws.Store { // This test expects docker to be running in linux CI environments and fails if it's not if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { diff --git a/pkg/store/storage_provider/postgres/store.go b/pkg/store/storage_provider/postgres/store.go new file mode 100644 index 0000000..76a9c89 --- /dev/null +++ b/pkg/store/storage_provider/postgres/store.go @@ -0,0 +1,169 @@ +// Package postgres provides a PostgreSQL-backed implementation of storage_provider.Store. +package postgres + +import ( + "context" + "errors" + "fmt" + "net/url" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/storacha/go-ucanto/core/delegation" + "github.com/storacha/go-ucanto/did" + "github.com/storacha/sprue/pkg/store" + storageprovider "github.com/storacha/sprue/pkg/store/storage_provider" +) + +const defaultListLimit = 1000 + +type Store struct { + pool *pgxpool.Pool +} + +var _ storageprovider.Store = (*Store)(nil) + +func New(pool *pgxpool.Pool) *Store { + return &Store{pool: pool} +} + +func (s *Store) Initialize(ctx context.Context) error { return nil } + +func (s *Store) Put(ctx context.Context, endpoint url.URL, proof delegation.Delegation, weight int, replicationWeight *int) error { + proofStr, err := delegation.Format(proof) + if err != nil { + return fmt.Errorf("formatting proof: %w", err) + } + + now := time.Now().UTC() + _, err = s.pool.Exec(ctx, ` + INSERT INTO storage_provider (provider, endpoint, proof, weight, replication_weight, inserted_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $6) + ON CONFLICT (provider) DO UPDATE + SET endpoint = EXCLUDED.endpoint, + proof = EXCLUDED.proof, + weight = EXCLUDED.weight, + replication_weight = EXCLUDED.replication_weight, + updated_at = EXCLUDED.updated_at + `, proof.Issuer().DID().String(), endpoint.String(), proofStr, weight, replicationWeight, now) + if err != nil { + return fmt.Errorf("storing storage provider: %w", err) + } + return nil +} + +func (s *Store) Get(ctx context.Context, providerID did.DID) (storageprovider.Record, error) { + row := s.pool.QueryRow(ctx, ` + SELECT provider, endpoint, proof, weight, replication_weight, inserted_at, updated_at + FROM storage_provider + WHERE provider = $1 + `, providerID.String()) + rec, err := scanRecord(row) + if errors.Is(err, pgx.ErrNoRows) { + return storageprovider.Record{}, storageprovider.ErrStorageProviderNotFound + } + if err != nil { + return storageprovider.Record{}, fmt.Errorf("getting storage provider: %w", err) + } + return rec, nil +} + +func (s *Store) Delete(ctx context.Context, providerID did.DID) error { + tag, err := s.pool.Exec(ctx, `DELETE FROM storage_provider WHERE provider = $1`, providerID.String()) + if err != nil { + return fmt.Errorf("deleting storage provider: %w", err) + } + if tag.RowsAffected() == 0 { + return storageprovider.ErrStorageProviderNotFound + } + return nil +} + +func (s *Store) List(ctx context.Context, options ...storageprovider.ListOption) (store.Page[storageprovider.Record], error) { + cfg := storageprovider.ListConfig{} + for _, opt := range options { + opt(&cfg) + } + limit := defaultListLimit + if cfg.Limit != nil && *cfg.Limit > 0 { + limit = *cfg.Limit + } + + args := []any{limit + 1} + query := ` + SELECT provider, endpoint, proof, weight, replication_weight, inserted_at, updated_at + FROM storage_provider + ` + if cfg.Cursor != nil { + args = append(args, *cfg.Cursor) + query += ` WHERE provider > $2` + } + query += ` ORDER BY provider ASC LIMIT $1` + + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return store.Page[storageprovider.Record]{}, fmt.Errorf("listing storage providers: %w", err) + } + defer rows.Close() + + records := make([]storageprovider.Record, 0, limit) + for rows.Next() { + rec, err := scanRecord(rows) + if err != nil { + return store.Page[storageprovider.Record]{}, err + } + records = append(records, rec) + } + if err := rows.Err(); err != nil { + return store.Page[storageprovider.Record]{}, fmt.Errorf("listing storage providers: %w", err) + } + + var cursor *string + if len(records) > limit { + last := records[limit-1].Provider.String() + cursor = &last + records = records[:limit] + } + return store.Page[storageprovider.Record]{Results: records, Cursor: cursor}, nil +} + +type rowScanner interface { + Scan(dest ...any) error +} + +func scanRecord(row rowScanner) (storageprovider.Record, error) { + var ( + providerStr string + endpointStr string + proofStr string + weight int + replicationWeight *int + insertedAt time.Time + updatedAt time.Time + ) + if err := row.Scan(&providerStr, &endpointStr, &proofStr, &weight, &replicationWeight, &insertedAt, &updatedAt); err != nil { + return storageprovider.Record{}, err + } + providerDID, err := did.Parse(providerStr) + if err != nil { + return storageprovider.Record{}, fmt.Errorf("parsing provider DID: %w", err) + } + endpoint, err := url.Parse(endpointStr) + if err != nil { + return storageprovider.Record{}, fmt.Errorf("parsing endpoint URL: %w", err) + } + proof, err := delegation.Parse(proofStr) + if err != nil { + return storageprovider.Record{}, fmt.Errorf("parsing proof: %w", err) + } + return storageprovider.Record{ + Provider: providerDID, + Endpoint: *endpoint, + Proof: proof, + Weight: weight, + ReplicationWeight: replicationWeight, + InsertedAt: insertedAt, + UpdatedAt: updatedAt, + }, nil +} diff --git a/pkg/store/storage_provider/storage_provider_test.go b/pkg/store/storage_provider/storage_provider_test.go index 17389fb..5ac6fbf 100644 --- a/pkg/store/storage_provider/storage_provider_test.go +++ b/pkg/store/storage_provider/storage_provider_test.go @@ -14,17 +14,19 @@ import ( storageprovider "github.com/storacha/sprue/pkg/store/storage_provider" storageprovideraws "github.com/storacha/sprue/pkg/store/storage_provider/aws" "github.com/storacha/sprue/pkg/store/storage_provider/memory" + storageproviderpostgres "github.com/storacha/sprue/pkg/store/storage_provider/postgres" "github.com/stretchr/testify/require" ) type StoreKind string const ( - Memory StoreKind = "memory" - AWS StoreKind = "aws" + Memory StoreKind = "memory" + AWS StoreKind = "aws" + Postgres StoreKind = "postgres" ) -var storeKinds = []StoreKind{Memory, AWS} +var storeKinds = []StoreKind{Memory, AWS, Postgres} func makeStore(t *testing.T, k StoreKind) storageprovider.Store { switch k { @@ -32,10 +34,25 @@ func makeStore(t *testing.T, k StoreKind) storageprovider.Store { return memory.New() case AWS: return createAWSStore(t) + case Postgres: + return createPostgresStore(t) } panic("unknown store kind") } +func createPostgresStore(t *testing.T) storageprovider.Store { + if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { + if !testutil.IsDockerAvailable(t) { + t.Fatalf("docker is expected in CI linux testing environments, but wasn't found") + } + } + if !testutil.IsDockerAvailable(t) { + t.SkipNow() + } + pool := testutil.CreatePostgres(t) + return storageproviderpostgres.New(pool) +} + func createAWSStore(t *testing.T) storageprovider.Store { // This test expects docker to be running in linux CI environments and fails if it's not if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { diff --git a/pkg/store/subscription/postgres/store.go b/pkg/store/subscription/postgres/store.go new file mode 100644 index 0000000..b6468a9 --- /dev/null +++ b/pkg/store/subscription/postgres/store.go @@ -0,0 +1,145 @@ +// Package postgres provides a PostgreSQL-backed implementation of subscription.Store. +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/ipfs/go-cid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/storacha/go-ucanto/did" + "github.com/storacha/sprue/pkg/store" + "github.com/storacha/sprue/pkg/store/subscription" +) + +const ( + defaultListLimit = 1000 + uniqueViolation = "23505" +) + +type Store struct { + pool *pgxpool.Pool +} + +var _ subscription.Store = (*Store)(nil) + +func New(pool *pgxpool.Pool) *Store { + return &Store{pool: pool} +} + +func (s *Store) Initialize(ctx context.Context) error { return nil } + +func (s *Store) Add(ctx context.Context, provider did.DID, subscriptionID string, customer did.DID, cause cid.Cid) error { + _, err := s.pool.Exec(ctx, ` + INSERT INTO subscription (subscription, provider, customer, cause, inserted_at) + VALUES ($1, $2, $3, $4, $5) + `, subscriptionID, provider.String(), customer.String(), cause.String(), time.Now().UTC()) + if err != nil { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == uniqueViolation { + return subscription.ErrSubscriptionExists + } + return fmt.Errorf("adding subscription: %w", err) + } + return nil +} + +func (s *Store) Get(ctx context.Context, provider did.DID, subscriptionID string) (subscription.Record, error) { + row := s.pool.QueryRow(ctx, ` + SELECT subscription, provider, customer, cause, inserted_at + FROM subscription + WHERE subscription = $1 AND provider = $2 + `, subscriptionID, provider.String()) + rec, err := scanRecord(row) + if errors.Is(err, pgx.ErrNoRows) { + return subscription.Record{}, subscription.ErrSubscriptionNotFound + } + if err != nil { + return subscription.Record{}, fmt.Errorf("getting subscription: %w", err) + } + return rec, nil +} + +func (s *Store) ListByProviderAndCustomer(ctx context.Context, provider did.DID, customer did.DID, options ...subscription.ListByProviderAndCustomerOption) (store.Page[subscription.Record], error) { + cfg := subscription.ListByProviderAndCustomerConfig{} + for _, opt := range options { + opt(&cfg) + } + limit := defaultListLimit + if cfg.Limit != nil && *cfg.Limit > 0 { + limit = *cfg.Limit + } + + args := []any{customer.String(), provider.String(), limit + 1} + query := ` + SELECT subscription, provider, customer, cause, inserted_at + FROM subscription + WHERE customer = $1 AND provider = $2 + ` + if cfg.Cursor != nil { + args = append(args, *cfg.Cursor) + query += ` AND subscription > $4` + } + query += ` ORDER BY subscription ASC LIMIT $3` + + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return store.Page[subscription.Record]{}, fmt.Errorf("listing subscriptions: %w", err) + } + defer rows.Close() + + records := make([]subscription.Record, 0, limit) + for rows.Next() { + rec, err := scanRecord(rows) + if err != nil { + return store.Page[subscription.Record]{}, err + } + records = append(records, rec) + } + if err := rows.Err(); err != nil { + return store.Page[subscription.Record]{}, fmt.Errorf("listing subscriptions: %w", err) + } + + var cursor *string + if len(records) > limit { + last := records[limit-1].Subscription + cursor = &last + records = records[:limit] + } + return store.Page[subscription.Record]{Results: records, Cursor: cursor}, nil +} + +type rowScanner interface { + Scan(dest ...any) error +} + +func scanRecord(row rowScanner) (subscription.Record, error) { + var subscriptionID, providerStr, customerStr, causeStr string + var insertedAt time.Time + if err := row.Scan(&subscriptionID, &providerStr, &customerStr, &causeStr, &insertedAt); err != nil { + return subscription.Record{}, err + } + provider, err := did.Parse(providerStr) + if err != nil { + return subscription.Record{}, fmt.Errorf("parsing provider DID: %w", err) + } + customer, err := did.Parse(customerStr) + if err != nil { + return subscription.Record{}, fmt.Errorf("parsing customer DID: %w", err) + } + cause, err := cid.Parse(causeStr) + if err != nil { + return subscription.Record{}, fmt.Errorf("parsing cause CID: %w", err) + } + return subscription.Record{ + Subscription: subscriptionID, + Provider: provider, + Customer: customer, + Cause: cause, + InsertedAt: insertedAt, + }, nil +} diff --git a/pkg/store/subscription/subscription_test.go b/pkg/store/subscription/subscription_test.go index 6b4fe36..b5322d6 100644 --- a/pkg/store/subscription/subscription_test.go +++ b/pkg/store/subscription/subscription_test.go @@ -12,17 +12,19 @@ import ( "github.com/storacha/sprue/pkg/store/subscription" subscriptionaws "github.com/storacha/sprue/pkg/store/subscription/aws" "github.com/storacha/sprue/pkg/store/subscription/memory" + subscriptionpostgres "github.com/storacha/sprue/pkg/store/subscription/postgres" "github.com/stretchr/testify/require" ) type StoreKind string const ( - Memory StoreKind = "memory" - AWS StoreKind = "aws" + Memory StoreKind = "memory" + AWS StoreKind = "aws" + Postgres StoreKind = "postgres" ) -var storeKinds = []StoreKind{Memory, AWS} +var storeKinds = []StoreKind{Memory, AWS, Postgres} func makeStore(t *testing.T, k StoreKind) subscription.Store { switch k { @@ -30,10 +32,25 @@ func makeStore(t *testing.T, k StoreKind) subscription.Store { return memory.New() case AWS: return createAWSStore(t) + case Postgres: + return createPostgresStore(t) } panic("unknown store kind") } +func createPostgresStore(t *testing.T) subscription.Store { + if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { + if !testutil.IsDockerAvailable(t) { + t.Fatalf("docker is expected in CI linux testing environments, but wasn't found") + } + } + if !testutil.IsDockerAvailable(t) { + t.SkipNow() + } + pool := testutil.CreatePostgres(t) + return subscriptionpostgres.New(pool) +} + func createAWSStore(t *testing.T) subscription.Store { // This test expects docker to be running in linux CI environments and fails if it's not if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { diff --git a/pkg/store/upload/postgres/store.go b/pkg/store/upload/postgres/store.go new file mode 100644 index 0000000..65cd27c --- /dev/null +++ b/pkg/store/upload/postgres/store.go @@ -0,0 +1,275 @@ +// Package postgres provides a PostgreSQL-backed implementation of upload.Store. +// +// Unlike the AWS backend (which pushes large shard lists to S3 to work around +// DynamoDB's 400 KB item-size limit), Postgres stores shards in a dedicated +// upload_shard table with no size restriction. +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/ipfs/go-cid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/storacha/go-ucanto/did" + "github.com/storacha/sprue/pkg/store" + "github.com/storacha/sprue/pkg/store/upload" +) + +const ( + defaultListLimit = 1000 + maxShardsPerPage = 1000 + defaultShardPageLimit = 1000 +) + +type Store struct { + pool *pgxpool.Pool +} + +var _ upload.Store = (*Store)(nil) + +func New(pool *pgxpool.Pool) *Store { + return &Store{pool: pool} +} + +func (s *Store) Initialize(ctx context.Context) error { return nil } + +func (s *Store) Exists(ctx context.Context, space did.DID, root cid.Cid) (bool, error) { + var exists bool + err := s.pool.QueryRow(ctx, ` + SELECT EXISTS ( + SELECT 1 FROM upload WHERE space = $1 AND root = $2 + ) + `, space.String(), root.String()).Scan(&exists) + if err != nil { + return false, fmt.Errorf("checking upload existence: %w", err) + } + return exists, nil +} + +func (s *Store) Get(ctx context.Context, space did.DID, root cid.Cid) (upload.UploadRecord, error) { + row := s.pool.QueryRow(ctx, ` + SELECT space, root, cause, inserted_at, updated_at + FROM upload + WHERE space = $1 AND root = $2 + `, space.String(), root.String()) + rec, err := scanRecord(row) + if errors.Is(err, pgx.ErrNoRows) { + return upload.UploadRecord{}, upload.ErrUploadNotFound + } + if err != nil { + return upload.UploadRecord{}, fmt.Errorf("getting upload: %w", err) + } + return rec, nil +} + +func (s *Store) Inspect(ctx context.Context, root cid.Cid) (upload.UploadInspectRecord, error) { + rows, err := s.pool.Query(ctx, `SELECT space FROM upload WHERE root = $1`, root.String()) + if err != nil { + return upload.UploadInspectRecord{}, fmt.Errorf("inspecting upload: %w", err) + } + defer rows.Close() + var spaces []did.DID + for rows.Next() { + var spaceStr string + if err := rows.Scan(&spaceStr); err != nil { + return upload.UploadInspectRecord{}, fmt.Errorf("scanning upload inspect row: %w", err) + } + spaceDID, err := did.Parse(spaceStr) + if err != nil { + return upload.UploadInspectRecord{}, fmt.Errorf("parsing space DID: %w", err) + } + spaces = append(spaces, spaceDID) + } + if err := rows.Err(); err != nil { + return upload.UploadInspectRecord{}, fmt.Errorf("iterating upload inspect rows: %w", err) + } + return upload.UploadInspectRecord{Spaces: spaces}, nil +} + +func (s *Store) List(ctx context.Context, space did.DID, options ...upload.ListOption) (store.Page[upload.UploadRecord], error) { + cfg := upload.ListConfig{} + for _, o := range options { + o(&cfg) + } + limit := defaultListLimit + if cfg.Limit != nil && *cfg.Limit > 0 { + limit = *cfg.Limit + } + + args := []any{space.String(), limit + 1} + query := ` + SELECT space, root, cause, inserted_at, updated_at + FROM upload + WHERE space = $1 + ` + if cfg.Cursor != nil { + args = append(args, *cfg.Cursor) + query += ` AND root > $3` + } + query += ` ORDER BY root ASC LIMIT $2` + + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return store.Page[upload.UploadRecord]{}, fmt.Errorf("listing uploads: %w", err) + } + defer rows.Close() + + records := make([]upload.UploadRecord, 0, limit) + for rows.Next() { + rec, err := scanRecord(rows) + if err != nil { + return store.Page[upload.UploadRecord]{}, err + } + records = append(records, rec) + } + if err := rows.Err(); err != nil { + return store.Page[upload.UploadRecord]{}, fmt.Errorf("iterating uploads: %w", err) + } + + var cursor *string + if len(records) > limit { + last := records[limit-1].Root.String() + cursor = &last + records = records[:limit] + } + return store.Page[upload.UploadRecord]{Results: records, Cursor: cursor}, nil +} + +func (s *Store) ListShards(ctx context.Context, space did.DID, root cid.Cid, options ...upload.ListShardsOption) (store.Page[cid.Cid], error) { + cfg := upload.ListShardsConfig{} + for _, o := range options { + o(&cfg) + } + limit := defaultShardPageLimit + if cfg.Limit != nil && *cfg.Limit > 0 { + limit = *cfg.Limit + } + if limit > maxShardsPerPage { + limit = maxShardsPerPage + } + + args := []any{space.String(), root.String(), limit + 1} + query := ` + SELECT shard FROM upload_shard + WHERE space = $1 AND root = $2 + ` + if cfg.Cursor != nil { + args = append(args, *cfg.Cursor) + query += ` AND shard > $4` + } + query += ` ORDER BY shard ASC LIMIT $3` + + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return store.Page[cid.Cid]{}, fmt.Errorf("listing shards: %w", err) + } + defer rows.Close() + + shards := make([]cid.Cid, 0, limit) + for rows.Next() { + var shardStr string + if err := rows.Scan(&shardStr); err != nil { + return store.Page[cid.Cid]{}, fmt.Errorf("scanning shard: %w", err) + } + shard, err := cid.Parse(shardStr) + if err != nil { + return store.Page[cid.Cid]{}, fmt.Errorf("parsing shard CID: %w", err) + } + shards = append(shards, shard) + } + if err := rows.Err(); err != nil { + return store.Page[cid.Cid]{}, fmt.Errorf("iterating shards: %w", err) + } + + var cursor *string + if len(shards) > limit { + last := shards[limit-1].String() + cursor = &last + shards = shards[:limit] + } + return store.Page[cid.Cid]{Results: shards, Cursor: cursor}, nil +} + +func (s *Store) Remove(ctx context.Context, space did.DID, root cid.Cid) error { + tag, err := s.pool.Exec(ctx, `DELETE FROM upload WHERE space = $1 AND root = $2`, space.String(), root.String()) + if err != nil { + return fmt.Errorf("removing upload: %w", err) + } + if tag.RowsAffected() == 0 { + return upload.ErrUploadNotFound + } + return nil +} + +func (s *Store) Upsert(ctx context.Context, space did.DID, root cid.Cid, shards []cid.Cid, cause cid.Cid) error { + tx, err := s.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("beginning transaction: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + now := time.Now().UTC() + if _, err := tx.Exec(ctx, ` + INSERT INTO upload (space, root, cause, inserted_at, updated_at) + VALUES ($1, $2, $3, $4, $4) + ON CONFLICT (space, root) DO UPDATE + SET cause = EXCLUDED.cause, updated_at = EXCLUDED.updated_at + `, space.String(), root.String(), cause.String(), now); err != nil { + return fmt.Errorf("upserting upload: %w", err) + } + + for _, shard := range shards { + if _, err := tx.Exec(ctx, ` + INSERT INTO upload_shard (space, root, shard) + VALUES ($1, $2, $3) + ON CONFLICT (space, root, shard) DO NOTHING + `, space.String(), root.String(), shard.String()); err != nil { + return fmt.Errorf("upserting upload shard: %w", err) + } + } + + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("committing upload upsert: %w", err) + } + return nil +} + +type rowScanner interface { + Scan(dest ...any) error +} + +func scanRecord(row rowScanner) (upload.UploadRecord, error) { + var ( + spaceStr string + rootStr string + causeStr string + insertedAt time.Time + updatedAt time.Time + ) + if err := row.Scan(&spaceStr, &rootStr, &causeStr, &insertedAt, &updatedAt); err != nil { + return upload.UploadRecord{}, err + } + space, err := did.Parse(spaceStr) + if err != nil { + return upload.UploadRecord{}, fmt.Errorf("parsing space DID: %w", err) + } + root, err := cid.Parse(rootStr) + if err != nil { + return upload.UploadRecord{}, fmt.Errorf("parsing root CID: %w", err) + } + cause, err := cid.Parse(causeStr) + if err != nil { + return upload.UploadRecord{}, fmt.Errorf("parsing cause CID: %w", err) + } + return upload.UploadRecord{ + Space: space, + Root: root, + Cause: cause, + InsertedAt: insertedAt, + UpdatedAt: updatedAt, + }, nil +} diff --git a/pkg/store/upload/upload_test.go b/pkg/store/upload/upload_test.go index eb35018..63102f8 100644 --- a/pkg/store/upload/upload_test.go +++ b/pkg/store/upload/upload_test.go @@ -13,17 +13,19 @@ import ( "github.com/storacha/sprue/pkg/store/upload" "github.com/storacha/sprue/pkg/store/upload/aws" "github.com/storacha/sprue/pkg/store/upload/memory" + uploadpostgres "github.com/storacha/sprue/pkg/store/upload/postgres" "github.com/stretchr/testify/require" ) type StoreKind string const ( - Memory StoreKind = "memory" - AWS StoreKind = "aws" + Memory StoreKind = "memory" + AWS StoreKind = "aws" + Postgres StoreKind = "postgres" ) -var storeKinds = []StoreKind{Memory, AWS} +var storeKinds = []StoreKind{Memory, AWS, Postgres} func makeStore(t *testing.T, k StoreKind) upload.Store { switch k { @@ -31,10 +33,25 @@ func makeStore(t *testing.T, k StoreKind) upload.Store { return memory.New() case AWS: return createAWSStore(t) + case Postgres: + return createPostgresStore(t) } panic("unknown store kind") } +func createPostgresStore(t *testing.T) upload.Store { + if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" { + if !testutil.IsDockerAvailable(t) { + t.Fatalf("docker is expected in CI linux testing environments, but wasn't found") + } + } + if !testutil.IsDockerAvailable(t) { + t.SkipNow() + } + pool := testutil.CreatePostgres(t) + return uploadpostgres.New(pool) +} + func createAWSStore(t *testing.T) *aws.Store { // This test expects docker to be running in linux CI environments and fails if it's not if testutil.IsRunningInCI(t) && runtime.GOOS == "linux" {