diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index c4d71d7d..00000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: Deploy - -on: - push: - branches: - - main - tags: - - "*" - paths: - - ".github/workflows/deploy.yml" - - ".github/workflows/terraform.yml" - - "cmd/**" - - "deploy/**" - - "internal/**" - - "pkg/**" - pull_request: - branches: [main] - workflow_run: - workflows: [Releaser] - types: [completed] - branches: [main] - workflow_dispatch: - inputs: - environment: - type: choice - description: Environment - options: - - staging - - production - -permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout - -jobs: - # always deploy to staging - staging: - uses: ./.github/workflows/terraform.yml - with: - env: staging - workspace: staging - apply: ${{ github.event_name != 'pull_request' }} - network: staging - indexing-service-did: ${{ vars.STAGING_INDEXING_SERVICE_DID }} - indexing-service-url: ${{ vars.STAGING_INDEXING_SERVICE_URL }} - principal-mapping: ${{ vars.STAGING_PRINCIPAL_MAPPING }} - blob-bucket-key-pattern: ${{ vars.STAGING_BLOB_BUCKET_KEY_PATTERN }} - use-external-blob-bucket: ${{ vars.STAGING_USE_EXTERNAL_BLOB_BUCKET == 'true' }} - external-blob-bucket-endpoint: ${{ vars.STAGING_EXTERNAL_BLOB_BUCKET_ENDPOINT }} - external-blob-bucket-region: ${{ vars.STAGING_EXTERNAL_BLOB_BUCKET_REGION }} - external-blob-bucket-name: ${{ vars.STAGING_EXTERNAL_BLOB_BUCKET_NAME }} - external-blob-bucket-domain: ${{ vars.STAGING_EXTERNAL_BLOB_BUCKET_DOMAIN }} - ipni-announce-urls: ${{ vars.STAGING_IPNI_ANNOUNCE_URLS }} - secrets: - aws-account-id: ${{ secrets.STAGING_AWS_ACCOUNT_ID }} - aws-region: ${{ secrets.STAGING_AWS_REGION }} - region: ${{ secrets.STAGING_AWS_REGION }} - allowed-account-ids: ${{ secrets.STAGING_ALLOWED_ACCOUNT_IDS }} - private-key: ${{ secrets.STAGING_PRIVATE_KEY }} - indexing-service-proof: ${{ secrets.STAGING_INDEXING_SERVICE_PROOF }} - external-blob-bucket-access-key-id: ${{ secrets.STAGING_EXTERNAL_BLOB_BUCKET_ACCESS_KEY_ID }} - external-blob-bucket-secret-access-key: ${{ secrets.STAGING_EXTERNAL_BLOB_BUCKET_SECRET_ACCESS_KEY }} - sentry-dsn: ${{ secrets.SENTRY_DSN }} - - # deploy to prod on new releases - production: - if: ${{ (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') || (github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'production') }} - uses: ./.github/workflows/terraform.yml - with: - env: production - workspace: prod - apply: true - network: prod - indexing-service-did: ${{ vars.PROD_INDEXING_SERVICE_DID }} - indexing-service-url: ${{ vars.PROD_INDEXING_SERVICE_URL }} - principal-mapping: ${{ vars.PROD_PRINCIPAL_MAPPING }} - blob-bucket-key-pattern: ${{ vars.PROD_BLOB_BUCKET_KEY_PATTERN }} - use-external-blob-bucket: ${{ vars.PROD_USE_EXTERNAL_BLOB_BUCKET == 'true' }} - external-blob-bucket-endpoint: ${{ vars.PROD_EXTERNAL_BLOB_BUCKET_ENDPOINT }} - external-blob-bucket-region: ${{ vars.PROD_EXTERNAL_BLOB_BUCKET_REGION }} - external-blob-bucket-name: ${{ vars.PROD_EXTERNAL_BLOB_BUCKET_NAME }} - external-blob-bucket-domain: ${{ vars.PROD_EXTERNAL_BLOB_BUCKET_DOMAIN }} - ipni-announce-urls: ${{ vars.PROD_IPNI_ANNOUNCE_URLS }} - secrets: - aws-account-id: ${{ secrets.PROD_AWS_ACCOUNT_ID }} - aws-region: ${{ secrets.PROD_AWS_REGION }} - region: ${{ secrets.PROD_AWS_REGION }} - allowed-account-ids: ${{ secrets.PROD_ALLOWED_ACCOUNT_IDS }} - private-key: ${{ secrets.PROD_PRIVATE_KEY }} - indexing-service-proof: ${{ secrets.PROD_INDEXING_SERVICE_PROOF }} - external-blob-bucket-access-key-id: ${{ secrets.PROD_EXTERNAL_BLOB_BUCKET_ACCESS_KEY_ID }} - external-blob-bucket-secret-access-key: ${{ secrets.PROD_EXTERNAL_BLOB_BUCKET_SECRET_ACCESS_KEY }} - sentry-dsn: ${{ secrets.SENTRY_DSN }} diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml deleted file mode 100644 index e836145d..00000000 --- a/.github/workflows/terraform.yml +++ /dev/null @@ -1,133 +0,0 @@ -name: Terraform - -on: - workflow_call: - inputs: - env: - required: true - type: string - workspace: - required: true - type: string - apply: - required: true - type: boolean - network: - required: true - type: string - indexing-service-did: - required: false - type: string - indexing-service-url: - required: false - type: string - principal-mapping: - required: false - type: string - blob-bucket-key-pattern: - required: false - type: string - use-external-blob-bucket: - required: false - type: boolean - external-blob-bucket-endpoint: - required: false - type: string - external-blob-bucket-region: - required: false - type: string - external-blob-bucket-name: - required: false - type: string - external-blob-bucket-domain: - required: false - type: string - ipni-announce-urls: - required: false - type: string - secrets: - aws-account-id: - required: true - allowed-account-ids: - required: true - aws-region: - required: true - region: - required: true - private-key: - required: true - indexing-service-proof: - required: true - external-blob-bucket-access-key-id: - required: false - external-blob-bucket-secret-access-key: - required: false - sentry-dsn: - required: false - -concurrency: - group: ${{ github.workflow }}-${{ inputs.workspace }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - AWS_ACCOUNT_ID: ${{ secrets.aws-account-id }} - AWS_REGION: ${{ secrets.aws-region }} - ENV: ${{ inputs.env }} - TF_WORKSPACE: ${{ inputs.workspace }} - TF_VAR_region: ${{ secrets.region }} - TF_VAR_allowed_account_ids: ${{ secrets.allowed-account-ids }} - TF_VAR_private_key: ${{ secrets.private-key }} - TF_VAR_network: ${{ inputs.network }} - TF_VAR_indexing_service_did: ${{ inputs.indexing-service-did }} - TF_VAR_indexing_service_url: ${{ inputs.indexing-service-url }} - TF_VAR_indexing_service_proof: ${{ secrets.indexing-service-proof }} - TF_VAR_principal_mapping: ${{ inputs.principal-mapping }} - TF_VAR_blob_bucket_key_pattern: ${{ inputs.blob-bucket-key-pattern }} - TF_VAR_use_external_blob_bucket: ${{ inputs.use-external-blob-bucket }} - TF_VAR_external_blob_bucket_endpoint: ${{ inputs.external-blob-bucket-endpoint }} - TF_VAR_external_blob_bucket_region: ${{ inputs.external-blob-bucket-region }} - TF_VAR_external_blob_bucket_name: ${{ inputs.external-blob-bucket-name }} - TF_VAR_external_blob_bucket_domain: ${{ inputs.external-blob-bucket-domain }} - TF_VAR_external_blob_bucket_access_key_id: ${{ secrets.external-blob-bucket-access-key-id }} - TF_VAR_external_blob_bucket_secret_access_key: ${{ secrets.external-blob-bucket-secret-access-key }} - TF_VAR_sentry_dsn: ${{ secrets.sentry-dsn }} - TF_VAR_ipni_announce_urls: ${{ inputs.ipni-announce-urls }} - -permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout - -jobs: - terraform: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-region: ${{ env.AWS_REGION }} - role-to-assume: arn:aws:iam::${{ env.AWS_ACCOUNT_ID }}:role/terraform-ci - - - uses: opentofu/setup-opentofu@v1 - - uses: actions/setup-go@v5 - - - name: Tofu Init - run: | - tofu -chdir="deploy/app" init - - - name: Build Go Apps - run: | - cd deploy - touch .env - make lambdas - - - name: Terraform Plan - if: ${{ !inputs.apply }} - run: | - tofu -chdir="deploy/app" plan - - - name: Terraform Apply - if: ${{ inputs.apply }} - run: | - tofu -chdir="deploy/app" apply -input=false --auto-approve diff --git a/Makefile b/Makefile index efeaa696..37e78280 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ DATE=$(shell date -u -Iseconds) GOFLAGS=-ldflags="-X github.com/storacha/piri/pkg/build.version=$(VERSION) -X github.com/storacha/piri/pkg/build.Commit=$(COMMIT) -X github.com/storacha/piri/pkg/build.Date=$(DATE) -X github.com/storacha/piri/pkg/build.BuiltBy=make" TAGS?= -.PHONY: all build install test clean calibnet mockgen check-docs-links +.PHONY: all build test clean all: build @@ -20,44 +20,8 @@ piri: FORCE FORCE: -install: - go install ./cmd/storage - test: go test ./... clean: - rm -f ./piri - -mockgen: - mockgen -source=./pkg/pdp/aggregator/interface.go -destination=./internal/mocks/aggregator.go -package=mocks - mockgen -source=./pkg/pdp/types/api.go -destination=./internal/mocks/pdp_api.go -package=mocks - mockgen -source=./internal/ipldstore/ipldstore.go -destination=./internal/mocks/ipldstore.go -package=mocks - mockgen -source=./pkg/pdp/aggregator/steps.go -destination=./internal/mocks/steps.go -package=mocks - mockgen -destination=./internal/mocks/sender_eth_client.go -package=mocks github.com/storacha/piri/pkg/pdp/tasks SenderETHClient - mockgen -destination=./internal/mocks/message_watcher_eth_client.go -package=mocks github.com/storacha/piri/pkg/pdp/tasks MessageWatcherEthClient - mockgen -destination=./internal/mocks/contract_backend.go -package=mocks github.com/ethereum/go-ethereum/accounts/abi/bind ContractBackend - mockgen -source=./pkg/pdp/smartcontracts/contract.go -destination=./pkg/pdp/smartcontracts/mocks/pdp.go -package=mocks - -# Contract generation targets -.PHONY: generate-contracts clean-contracts - -generate-contracts: - cd pkg/pdp/smartcontracts && ./generate.sh - -clean-contracts: - rm -rf pkg/pdp/smartcontracts/abis - rm -rf pkg/pdp/smartcontracts/bindings - rm -f pkg/pdp/smartcontracts/mocks/*.go - -mockgen-contracts: generate-contracts - mockgen -source=./pkg/pdp/smartcontracts/contract.go -destination=./pkg/pdp/smartcontracts/mocks/pdp.go -package=mocks - - -# special target that sets the calibnet tag and invokes build -calibnet: TAGS=-tags calibnet -calibnet: build - -# Check for broken links in documentation -check-docs-links: - @./scripts/check-docs-links.sh + rm -f ./piri \ No newline at end of file diff --git a/cmd/cliutil/util.go b/cmd/cliutil/util.go index c3779520..5ab43709 100644 --- a/cmd/cliutil/util.go +++ b/cmd/cliutil/util.go @@ -3,14 +3,8 @@ package cliutil import ( "fmt" "io" - "net/url" - "os" - "path" "github.com/labstack/gommon/color" - "github.com/multiformats/go-multiaddr" - "github.com/spf13/cobra" - "github.com/storacha/go-ucanto/core/delegation" "github.com/storacha/go-ucanto/did" "github.com/storacha/piri/pkg/build" @@ -33,63 +27,3 @@ func PrintHero(w io.Writer, id did.DID) { color.Red("▀")+color.Red("▘", color.D), build.Version, id.String()) } - -func Mkdirp(dirpath ...string) (string, error) { - dir := path.Join(dirpath...) - err := os.MkdirAll(dir, 0755) - if err != nil { - return "", fmt.Errorf("creating directory: %s: %w", dir, err) - } - return dir, nil -} - -type UCANServerConfig struct { - Host string - Port uint - DataDir string - PublicURL *url.URL - BlobAddr multiaddr.Multiaddr - IndexingServiceDID did.DID - IndexingServiceURL *url.URL - IndexingServiceProof delegation.Proof - UploadServiceDID did.DID - UploadServiceURL *url.URL - IPNIAnnounceURLs []url.URL -} - -func PrintUCANServerConfig(cmd *cobra.Command, cfg UCANServerConfig) { - cmd.Println("SERVER CONFIGURATION") - cmd.Println("--------------------") - cmd.Printf("Host: %s\n", cfg.Host) - cmd.Printf("Port: %d\n", cfg.Port) - cmd.Printf("Data Dir: %s\n", cfg.DataDir) - cmd.Printf("Public URL: %s\n", cfg.PublicURL) - if cfg.BlobAddr != nil { - cmd.Printf("Blob Addr: %s\n", cfg.BlobAddr) - } - - cmd.Println() - cmd.Println("SERVICES") - cmd.Println("--------") - cmd.Println("Indexing Service:") - cmd.Printf(" DID: %s\n", cfg.IndexingServiceDID) - cmd.Printf(" URL: %s\n", cfg.IndexingServiceURL) - cmd.Printf(" Proof Set: %t\n", cfg.IndexingServiceProof != delegation.Proof{}) - cmd.Println() - cmd.Println("Upload Service:") - cmd.Printf(" DID: %s\n", cfg.UploadServiceDID) - cmd.Printf(" URL: %s\n", cfg.UploadServiceURL) - - cmd.Println() - cmd.Println("IPNI ANNOUNCE URLS") - cmd.Println("------------------") - if len(cfg.IPNIAnnounceURLs) == 0 { - cmd.Println(" (none configured)") - } else { - for _, url := range cfg.IPNIAnnounceURLs { - cmd.Printf(" • %s\n", url.String()) - } - } - - cmd.Println() -} diff --git a/cmd/lambda/advertisementpublisher/main.go b/cmd/lambda/advertisementpublisher/main.go deleted file mode 100644 index 6c8bd2df..00000000 --- a/cmd/lambda/advertisementpublisher/main.go +++ /dev/null @@ -1,87 +0,0 @@ -package main - -import ( - "context" - "fmt" - "time" - - "github.com/aws/aws-lambda-go/events" - "github.com/labstack/gommon/log" - "github.com/libp2p/go-libp2p/core/crypto" - "github.com/multiformats/go-multiaddr" - "github.com/storacha/go-libstoracha/ipnipublisher/publisher" - awspublishingqueue "github.com/storacha/go-libstoracha/ipnipublisher/queue/aws" - "github.com/storacha/go-libstoracha/ipnipublisher/store" - "github.com/storacha/go-libstoracha/metadata" - "github.com/storacha/piri/cmd/lambda" - "github.com/storacha/piri/pkg/aws" -) - -const gracePeriod = time.Second - -func main() { - lambda.StartBatchSQSEventHandler(makeHandler) -} - -func makeHandler(cfg aws.Config) (lambda.SQSBatchEventHandler, error) { - sqsAdvertisementPublishingDecoder := awspublishingqueue.NewSQSAdvertisementPublishingDecoder() - ipniStore := aws.NewS3Store(cfg.Config, cfg.IPNIStoreBucket, cfg.IPNIStorePrefix, cfg.S3Options...) - chunkLinksTable := aws.NewDynamoProviderContextTable(cfg.Config, cfg.ChunkLinksTableName, cfg.DynamoOptions...) - metadataTable := aws.NewDynamoProviderContextTable(cfg.Config, cfg.MetadataTableName, cfg.DynamoOptions...) - publisherStore := store.NewPublisherStore(ipniStore, chunkLinksTable, metadataTable, store.WithMetadataContext(metadata.MetadataContext)) - priv, err := crypto.UnmarshalEd25519PrivateKey(cfg.Signer.Raw()) - if err != nil { - return nil, fmt.Errorf("unmarshaling private key: %w", err) - } - announceAddr, err := multiaddr.NewMultiaddr(cfg.IPNIPublisherAnnounceAddress) - if err != nil { - return nil, fmt.Errorf("parsing announce multiaddr: %w", err) - } - - opts := []publisher.Option{publisher.WithAnnounceAddrs(announceAddr.String())} - for _, url := range cfg.IPNIAnnounceURLs { - opts = append(opts, publisher.WithDirectAnnounce(url.String())) - } - advertisementPublisher, err := publisher.NewAdvertisementPublisher( - priv, publisherStore, - opts..., - ) - if err != nil { - return nil, fmt.Errorf("creating IPNI publisher instance: %w", err) - } - return func(ctx context.Context, sqsEvent events.SQSEvent) (events.SQSEventResponse, error) { - deadline, ok := ctx.Deadline() - if ok { - graceDeadline := deadline.Add(-gracePeriod) - // if graceful shutdown time is after now then we can apply new deadline - if graceDeadline.After(time.Now()) { - dctx, cancel := context.WithDeadline(ctx, graceDeadline) - defer cancel() - ctx = dctx - } - } - - failures := make([]events.SQSBatchItemFailure, 0, len(sqsEvent.Records)) - for _, msg := range sqsEvent.Records { - ad, err := sqsAdvertisementPublishingDecoder.DecodeMessage(ctx, msg.ReceiptHandle, msg.Body) - if err != nil { - failures = append(failures, events.SQSBatchItemFailure{ - ItemIdentifier: msg.MessageId, - }) - continue - } - advertisementPublisher.AddToBatch(ad.Job) - } - _, err := advertisementPublisher.Commit(ctx) - if err != nil { - log.Errorf("failed to commit advertisement batch: %s", err) - failures = make([]events.SQSBatchItemFailure, 0, len(sqsEvent.Records)) - for _, msg := range sqsEvent.Records { - failures = append(failures, events.SQSBatchItemFailure{ - ItemIdentifier: msg.MessageId, - }) - } - } - return events.SQSEventResponse{BatchItemFailures: failures}, nil - }, nil -} diff --git a/cmd/lambda/getclaim/main.go b/cmd/lambda/getclaim/main.go deleted file mode 100644 index 5af60856..00000000 --- a/cmd/lambda/getclaim/main.go +++ /dev/null @@ -1,34 +0,0 @@ -package main - -import ( - "net/http" - - logging "github.com/ipfs/go-log/v2" - - "github.com/storacha/piri/cmd/lambda" - "github.com/storacha/piri/internal/telemetry" - "github.com/storacha/piri/pkg/aws" - "github.com/storacha/piri/pkg/service/claims" -) - -var log = logging.Logger("lambda/getclaim") - -func main() { - lambda.StartHTTPHandler(makeHandler) -} - -func makeHandler(cfg aws.Config) (http.Handler, error) { - service, err := aws.Construct(cfg) - if err != nil { - return nil, err - } - - handler := claims.NewHandler(service.Claims().Store()) - return telemetry.NewErrorReportingHandler(func(w http.ResponseWriter, r *http.Request) error { - err := handler(aws.NewHandlerContext(w, r)) - if err != nil { - log.Error(err) - } - return err - }), nil -} diff --git a/cmd/lambda/getroot/main.go b/cmd/lambda/getroot/main.go deleted file mode 100644 index ee250229..00000000 --- a/cmd/lambda/getroot/main.go +++ /dev/null @@ -1,17 +0,0 @@ -package main - -import ( - "net/http" - - "github.com/storacha/piri/cmd/lambda" - "github.com/storacha/piri/pkg/aws" - "github.com/storacha/piri/pkg/server" -) - -func main() { - lambda.StartHTTPHandler(makeHandler) -} - -func makeHandler(cfg aws.Config) (http.Handler, error) { - return server.NewHandler(cfg.Signer), nil -} diff --git a/cmd/lambda/postad/main.go b/cmd/lambda/postad/main.go deleted file mode 100644 index bc1efe8e..00000000 --- a/cmd/lambda/postad/main.go +++ /dev/null @@ -1,130 +0,0 @@ -package main - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - - "github.com/ipld/go-ipld-prime" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipni/go-libipni/dagsync/ipnisync/head" - "github.com/ipni/go-libipni/ingest/schema" - "github.com/libp2p/go-libp2p/core/crypto" - "github.com/multiformats/go-multihash" - - "github.com/ipfs/go-cid" - - "github.com/storacha/go-libstoracha/ipnipublisher/store" - "github.com/storacha/go-libstoracha/metadata" - "github.com/storacha/piri/cmd/lambda" - "github.com/storacha/piri/pkg/aws" -) - -func main() { - lambda.StartHTTPHandler(makeHandler) -} - -func makeHandler(cfg aws.Config) (http.Handler, error) { - sk, err := crypto.UnmarshalEd25519PrivateKey(cfg.Signer.Raw()) - if err != nil { - return nil, err - } - ipniStore := aws.NewS3Store(cfg.Config, cfg.IPNIStoreBucket, cfg.IPNIStorePrefix, cfg.S3Options...) - chunkLinksTable := aws.NewDynamoProviderContextTable(cfg.Config, cfg.ChunkLinksTableName, cfg.DynamoOptions...) - metadataTable := aws.NewDynamoProviderContextTable(cfg.Config, cfg.MetadataTableName, cfg.DynamoOptions...) - publisherStore := store.NewPublisherStore(ipniStore, chunkLinksTable, metadataTable, store.WithMetadataContext(metadata.MetadataContext)) - - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ad, err := decodeAdvert(r.Body) - if err != nil { - http.Error(w, fmt.Sprintf("decoding advert: %s", err.Error()), http.StatusBadRequest) - return - } - - if err := validateAdvertSig(sk, ad); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - adlink, err := publishAdvert(r.Context(), sk, publisherStore, ad) - if err != nil { - http.Error(w, fmt.Sprintf("publishing advert: %s", err.Error()), http.StatusInternalServerError) - return - } - - out, err := json.Marshal(adlink) - if err != nil { - http.Error(w, fmt.Sprintf("marshaling JSON: %s", err.Error()), http.StatusInternalServerError) - return - } - w.Write(out) - }), nil -} - -// ensures the advert came from this node originally -func validateAdvertSig(sk crypto.PrivKey, ad schema.Advertisement) error { - sigBytes := ad.Signature - err := ad.Sign(sk) - if err != nil { - return fmt.Errorf("signing advert: %w", err) - } - if !bytes.Equal(sigBytes, ad.Signature) { - return errors.New("advert was not created by this node") - } - return nil -} - -// assumed in DAG-JSON encoding -func decodeAdvert(r io.Reader) (schema.Advertisement, error) { - advBytes, err := io.ReadAll(r) - if err != nil { - return schema.Advertisement{}, err - } - - adLink, err := cid.V1Builder{ - Codec: cid.DagJSON, - MhType: multihash.SHA2_256, - }.Sum(advBytes) - if err != nil { - return schema.Advertisement{}, err - } - - return schema.BytesToAdvertisement(adLink, advBytes) -} - -func publishAdvert(ctx context.Context, sk crypto.PrivKey, store store.PublisherStore, ad schema.Advertisement) (ipld.Link, error) { - prevHead, err := store.Head(ctx) - if err != nil { - return nil, err - } - - ad.PreviousID = prevHead.Head - - // Sign the advertisement. - if err = ad.Sign(sk); err != nil { - return nil, fmt.Errorf("signing advert: %w", err) - } - - if err := ad.Validate(); err != nil { - return nil, fmt.Errorf("validating advert: %w", err) - } - - link, err := store.PutAdvert(ctx, ad) - if err != nil { - return nil, fmt.Errorf("putting advert: %w", err) - } - - head, err := head.NewSignedHead(link.(cidlink.Link).Cid, "/indexer/ingest/mainnet", sk) - if err != nil { - return nil, fmt.Errorf("signing head: %w", err) - } - if _, err := store.ReplaceHead(ctx, prevHead, head); err != nil { - return nil, fmt.Errorf("replacing head: %w", err) - } - - return link, nil -} diff --git a/cmd/lambda/postad/main_test.go b/cmd/lambda/postad/main_test.go deleted file mode 100644 index 72e99d3a..00000000 --- a/cmd/lambda/postad/main_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package main - -import ( - "encoding/json" - "os" - "slices" - "testing" - - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/sync" - "github.com/ipni/go-libipni/ingest/schema" - "github.com/libp2p/go-libp2p/core/crypto" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/multiformats/go-multiaddr" - "github.com/multiformats/go-multihash" - "github.com/storacha/go-libstoracha/ipnipublisher/publisher" - "github.com/storacha/go-libstoracha/ipnipublisher/store" - "github.com/storacha/go-libstoracha/metadata" - "github.com/storacha/go-libstoracha/testutil" - "github.com/stretchr/testify/require" -) - -func TestValidateAdvertSig(t *testing.T) { - sk0, _, err := crypto.GenerateEd25519Key(nil) - require.NoError(t, err) - - ad := schema.Advertisement{ - Entries: testutil.RandomCID(t), - } - err = ad.Sign(sk0) - require.NoError(t, err) - - err = validateAdvertSig(sk0, ad) - require.NoError(t, err) - - sk1, _, err := crypto.GenerateEd25519Key(nil) - require.NoError(t, err) - - err = validateAdvertSig(sk1, ad) - require.Error(t, err) - require.Contains(t, err.Error(), "advert was not created by this node") -} - -func TestDecodeAdvert(t *testing.T) { - f, err := os.Open("./testdata/advert.json") - require.NoError(t, err) - defer f.Close() - - ad, err := decodeAdvert(f) - require.NoError(t, err) - require.NotEmpty(t, ad) -} - -func TestPublishAdvert(t *testing.T) { - f, err := os.Open("./testdata/advert.json") - require.NoError(t, err) - defer f.Close() - - ad, err := decodeAdvert(f) - require.NoError(t, err) - - sk, _, err := crypto.GenerateEd25519Key(nil) - require.NoError(t, err) - - publisherStore := store.FromDatastore( - sync.MutexWrap(datastore.NewMapDatastore()), - store.WithMetadataContext(metadata.MetadataContext), - ) - - // we need at least one advert already to be able to call [publishAdvert] - pub, err := publisher.New(sk, publisherStore) - require.NoError(t, err) - - ctxID := string(testutil.RandomBytes(t, 32)) - md := metadata.MetadataContext.New() - providerInfo := peer.AddrInfo{ - ID: testutil.RandomPeer(t), - Addrs: []multiaddr.Multiaddr{testutil.RandomMultiaddr(t)}, - } - digests := []multihash.Multihash{testutil.RandomMultihash(t)} - - _, err = pub.Publish(t.Context(), providerInfo, ctxID, slices.Values(digests), md) - require.NoError(t, err) - - adlink, err := publishAdvert(t.Context(), sk, publisherStore, ad) - require.NoError(t, err) - - out, err := json.Marshal(adlink) - require.NoError(t, err) - t.Log(string(out)) -} diff --git a/cmd/lambda/postad/testdata/advert.json b/cmd/lambda/postad/testdata/advert.json deleted file mode 100644 index f54346f3..00000000 --- a/cmd/lambda/postad/testdata/advert.json +++ /dev/null @@ -1 +0,0 @@ -{"Addresses":["/dns/carpark-prod-1.r2.w3s.link/https/http-path/%7Bblob%7D%2F%7Bblob%7D.blob","/dns/storage.storacha.network/https/http-path/claim%2F%7Bclaim%7D"],"ContextID":{"/":{"bytes":"EiASAsSvXfUwig/PZvP/Dxm6+2y3jy+14UK9BQklbvzWFQ"}},"Entries":{"/":"baguqeerakaxvfkwkrf4jj5rgdxyfiscxzhad56e3ljjqm26eplrpvkkkr52q"},"IsRm":false,"Metadata":{"/":{"bytes":"goD4AaJhY9gqWCUAAXESIPDhyPfj3cdDAZ0kZPxjdn8vm7osEYsWGGLXAGa1yvuyYWUA"}},"PreviousID":{"/":"baguqeera4e4do723undcwzvm36b4hpysrwqqn4qulh6abxtuo4gmal3ydzyq"},"Provider":"12D3KooWLrikEsjt5wz326bRhCyEThRhJ936o13c5Ej7ttLbkxgp","Signature":{"/":{"bytes":"CiQIARIgpAr8IKGmGbPapkh2TN1oLDzSH97ru5UQJQFeRNpUm7kSGy9pbmRleGVyL2luZ2VzdC9hZFNpZ25hdHVyZRoiEiBfy7zkxCxSKrdwjw3Tb2uWsYE1nVKFefVp9VK/ZhoI0SpAMcdeqrdNW8OjHqxo2LHWZvCje6QTiPUodlFEZI4mK/qC0tvxDfd6mV7sl2uYJZtm4LTCLvrZBXtMX8CY05R9Dw"}}} \ No newline at end of file diff --git a/cmd/lambda/postroot/main.go b/cmd/lambda/postroot/main.go deleted file mode 100644 index bee45778..00000000 --- a/cmd/lambda/postroot/main.go +++ /dev/null @@ -1,47 +0,0 @@ -package main - -import ( - "net/http" - - ucanserver "github.com/storacha/go-ucanto/server" - - logging "github.com/ipfs/go-log/v2" - - "github.com/storacha/piri/cmd/lambda" - "github.com/storacha/piri/internal/telemetry" - "github.com/storacha/piri/pkg/aws" - "github.com/storacha/piri/pkg/principalresolver" - "github.com/storacha/piri/pkg/service/storage" -) - -var log = logging.Logger("lambda/postroot") - -func main() { - lambda.StartHTTPHandler(makeHandler) -} - -func makeHandler(cfg aws.Config) (http.Handler, error) { - service, err := aws.Construct(cfg) - if err != nil { - return nil, err - } - - presolv, err := principalresolver.NewMapResolver(cfg.PrincipalMapping) - if err != nil { - return nil, err - } - - server, err := storage.NewUCANServer(service, ucanserver.WithPrincipalResolver(presolv.ResolveDIDKey)) - if err != nil { - return nil, err - } - - handler := storage.NewHandler(server) - return telemetry.NewErrorReportingHandler(func(w http.ResponseWriter, r *http.Request) error { - err := handler(aws.NewHandlerContext(w, r)) - if err != nil { - log.Error(err) - } - return err - }), nil -} diff --git a/cmd/lambda/publisher/main.go b/cmd/lambda/publisher/main.go deleted file mode 100644 index 1a4c417e..00000000 --- a/cmd/lambda/publisher/main.go +++ /dev/null @@ -1,76 +0,0 @@ -package main - -import ( - "context" - "errors" - "time" - - "github.com/aws/aws-lambda-go/events" - "github.com/labstack/gommon/log" - "github.com/storacha/go-libstoracha/ipnipublisher/publisher" - "github.com/storacha/go-libstoracha/ipnipublisher/queue" - awspublishingqueue "github.com/storacha/go-libstoracha/ipnipublisher/queue/aws" - "github.com/storacha/go-libstoracha/ipnipublisher/store" - "github.com/storacha/go-libstoracha/metadata" - "github.com/storacha/piri/cmd/lambda" - "github.com/storacha/piri/pkg/aws" -) - -const gracePeriod = time.Second - -func main() { - lambda.StartBatchSQSEventHandler(makeHandler) -} - -func makeHandler(cfg aws.Config) (lambda.SQSBatchEventHandler, error) { - sqsPublishingDecoder := awspublishingqueue.NewSQSPublishingDecoder(cfg.Config, cfg.PublishingBucket) - ipniStore := aws.NewS3Store(cfg.Config, cfg.IPNIStoreBucket, cfg.IPNIStorePrefix, cfg.S3Options...) - chunkLinksTable := aws.NewDynamoProviderContextTable(cfg.Config, cfg.ChunkLinksTableName, cfg.DynamoOptions...) - metadataTable := aws.NewDynamoProviderContextTable(cfg.Config, cfg.MetadataTableName, cfg.DynamoOptions...) - publisherStore := store.NewPublisherStore(ipniStore, chunkLinksTable, metadataTable, store.WithMetadataContext(metadata.MetadataContext)) - advertisementPublishingQueue := awspublishingqueue.NewSQSAdvertisementPublishingQueue(cfg.Config, cfg.SQSAdvertisementPublishingQueueID) - advertismentQueuePublisher := queue.NewAdvertisementQueuePublisher(advertisementPublishingQueue, publisherStore) - - return func(ctx context.Context, sqsEvent events.SQSEvent) (events.SQSEventResponse, error) { - deadline, ok := ctx.Deadline() - if ok { - graceDeadline := deadline.Add(-gracePeriod) - // if graceful shutdown time is after now then we can apply new deadline - if graceDeadline.After(time.Now()) { - dctx, cancel := context.WithDeadline(ctx, graceDeadline) - defer cancel() - ctx = dctx - } - } - - failures := make([]events.SQSBatchItemFailure, 0, len(sqsEvent.Records)) - for _, msg := range sqsEvent.Records { - err := handleMessage(ctx, sqsPublishingDecoder, advertismentQueuePublisher, msg) - if err != nil { - failures = append(failures, events.SQSBatchItemFailure{ - ItemIdentifier: msg.MessageId, - }) - log.Errorf("unable to process message %s: %s", msg.MessageId, err.Error()) - } - } - return events.SQSEventResponse{BatchItemFailures: failures}, nil - }, nil -} - -func handleMessage(ctx context.Context, sqsPublishingDecoder *awspublishingqueue.SQSPublishingDecoder, publisher publisher.AsyncPublisher, msg events.SQSMessage) error { - job, err := sqsPublishingDecoder.DecodeMessage(ctx, msg.ReceiptHandle, msg.Body) - if err != nil { - return err - } - err = publisher.Publish(ctx, job.Job.ProviderInfo, job.Job.ContextID, job.Job.Digests, job.Job.Meta) - // Do not hold up the queue by re-attempting a cache job that times out. It is - // probably a big DAG and retrying is unlikely to subsequently succeed. - if errors.Is(err, context.DeadlineExceeded) { - log.Warnf("not retrying cache provider job for: %s error: %s", job.Job.ContextID, err) - return nil - } - if err != nil { - return err - } - return nil -} diff --git a/cmd/lambda/putblob/main.go b/cmd/lambda/putblob/main.go deleted file mode 100644 index be812011..00000000 --- a/cmd/lambda/putblob/main.go +++ /dev/null @@ -1,34 +0,0 @@ -package main - -import ( - "net/http" - - logging "github.com/ipfs/go-log/v2" - - "github.com/storacha/piri/cmd/lambda" - "github.com/storacha/piri/internal/telemetry" - "github.com/storacha/piri/pkg/aws" - "github.com/storacha/piri/pkg/service/blobs" -) - -var log = logging.Logger("lambda/putblob") - -func main() { - lambda.StartHTTPHandler(makeHandler) -} - -func makeHandler(cfg aws.Config) (http.Handler, error) { - service, err := aws.Construct(cfg) - if err != nil { - return nil, err - } - - handler := blobs.NewBlobPutHandler(service.Blobs().Presigner(), service.Blobs().Allocations(), service.Blobs().Store()) - return telemetry.NewErrorReportingHandler(func(w http.ResponseWriter, r *http.Request) error { - err := handler(aws.NewHandlerContext(w, r)) - if err != nil { - log.Error(err) - } - return err - }), nil -} diff --git a/cmd/lambda/start.go b/cmd/lambda/start.go deleted file mode 100644 index b106923f..00000000 --- a/cmd/lambda/start.go +++ /dev/null @@ -1,99 +0,0 @@ -package lambda - -import ( - "context" - "fmt" - "net/http" - - "github.com/aws/aws-lambda-go/events" - "github.com/aws/aws-lambda-go/lambda" - "github.com/awslabs/aws-lambda-go-api-proxy/httpadapter" - "github.com/storacha/piri/internal/telemetry" - "github.com/storacha/piri/pkg/aws" -) - -// SQSEventHandler is a function that handles SQS events, suitable to use as a lambda handler. -type SQSEventHandler func(context.Context, events.SQSEvent) error - -// SQSEventHandlerBuilder is a function that creates a SQSEventHandler from a config. -type SQSEventHandlerBuilder func(aws.Config) (SQSEventHandler, error) - -// StartSQSEventHandler starts a lambda handler that processes SQS events. -func StartSQSEventHandler(makeHandler SQSEventHandlerBuilder) { - ctx := context.Background() - cfg := aws.FromEnv(ctx) - telemetry.SetupErrorReporting(cfg.SentryDSN, cfg.SentryEnvironment) - - handler, err := makeHandler(cfg) - if err != nil { - telemetry.ReportError(ctx, err) - panic(err) - } - - lambda.StartWithOptions(instrumentSQSEventHandler(handler), lambda.WithContext(ctx)) -} - -// instrumentSQSEventHandler wraps a SQSEventHandler with error reporting. -func instrumentSQSEventHandler(handler SQSEventHandler) SQSEventHandler { - return func(ctx context.Context, sqsEvent events.SQSEvent) error { - err := handler(ctx, sqsEvent) - if err != nil { - telemetry.ReportError(ctx, err) - } - - return err - } -} - -// SQSBatchEventHandler is a function that handles SQS events, suitable to use as a lambda handler. -type SQSBatchEventHandler func(context.Context, events.SQSEvent) (events.SQSEventResponse, error) - -// SQSBatchEventHandlerBuilder is a function that creates a SQSBatchEventHandler from a config. -type SQSBatchEventHandlerBuilder func(aws.Config) (SQSBatchEventHandler, error) - -// StartBatchSQSEventHandler starts a lambda handler that processes SQS events. -func StartBatchSQSEventHandler(makeHandler SQSBatchEventHandlerBuilder) { - ctx := context.Background() - cfg := aws.FromEnv(ctx) - telemetry.SetupErrorReporting(cfg.SentryDSN, cfg.SentryEnvironment) - - handler, err := makeHandler(cfg) - if err != nil { - telemetry.ReportError(ctx, err) - panic(err) - } - - lambda.StartWithOptions(instrumentSQSBatchEventHandler(handler), lambda.WithContext(ctx)) -} - -// instrumentSQSBatchEventHandler wraps a SQSBatchEventHandler with error reporting. -func instrumentSQSBatchEventHandler(handler SQSBatchEventHandler) SQSBatchEventHandler { - return func(ctx context.Context, sqsEvent events.SQSEvent) (events.SQSEventResponse, error) { - failures, err := handler(ctx, sqsEvent) - if len(failures.BatchItemFailures) > 0 { - telemetry.ReportError(ctx, fmt.Errorf("handling batch SQS event failed: %v", failures.BatchItemFailures)) - } - if err != nil { - telemetry.ReportError(ctx, fmt.Errorf("handling batch SQS event failed: %v", err)) - } - return failures, err - } -} - -// HTTPHandlerBuilder is a function that creates a http.Handler from a config. -type HTTPHandlerBuilder func(aws.Config) (http.Handler, error) - -// StartHTTPHandler starts a lambda handler that processes HTTP requests. -func StartHTTPHandler(makeHandler HTTPHandlerBuilder) { - ctx := context.Background() - cfg := aws.FromEnv(ctx) - telemetry.SetupErrorReporting(cfg.SentryDSN, cfg.SentryEnvironment) - - handler, err := makeHandler(cfg) - if err != nil { - telemetry.ReportError(ctx, err) - panic(err) - } - - lambda.StartWithOptions(httpadapter.NewV2(handler).ProxyWithContext, lambda.WithContext(ctx)) -} diff --git a/deploy/.env.tpl b/deploy/.env.tpl deleted file mode 100644 index 3da37c3c..00000000 --- a/deploy/.env.tpl +++ /dev/null @@ -1,80 +0,0 @@ -################################################################################ -# REQUIRED -################################################################################ - -# Your name. Note: "staging" and "prod" are reserved for deployments from CI. -TF_WORKSPACE= -# Generate using CLI: `storage identity gen`. -TF_VAR_private_key= -# A delegation granting the DID of this storage node (public key corresponding -# to the value set in `TF_VAR_private_key`) "claim/cache" on an indexer node. -# Obtain from Storacha support. -TF_VAR_indexing_service_proof= -# Set to your AWS account ID. -TF_VAR_allowed_account_ids=["0"] -# Domain name to use for the deployment. Automatically prefixed with app name -# (see `TF_VAR_app`) and workspace name (unless workspace is "prod"). -# i.e. workspace.app.domain or app.domain if workspace == "prod". -TF_VAR_domain=storacha.network - -################################################################################ -# OPTIONAL -################################################################################ - -# Network config ############################################################### - -# The network setting sets the default values for some configurations, allowing -# the node to operate correctly in the given network. -TF_VAR_network=prod - -# AWS config ################################################################### - -# AWS region to deploy all resources. -TF_VAR_region=us-west-2 - -# Tags applied to AWS resources (useful for cost accounting). -TF_VAR_app=storage -TF_VAR_owner=storacha -TF_VAR_team=Storacha Engineering -TF_VAR_org=Storacha - -# Blob bucket config ########################################################### - -# Key pattern for blob bucket -TF_VAR_blob_bucket_key_pattern=blob/{blob} - -# Curio integration ############################################################ - -TF_VAR_use_pdp=false -TF_VAR_pdp_proofset=0 -TF_VAR_curio_url= - -# Sentry error reporting ####################################################### - -# Sentry DSN for error reporting. Obtain from sentry.io. -# Leave blank to disable error reporting. -TF_VAR_sentry_dsn= -# Sentry environment to use for error reporting. -# Defaults to the terraform workspace being used if not set. -TF_VAR_sentry_environment= - -# External (S3 compatible) blob bucket ######################################### - -TF_VAR_use_external_blob_bucket=false -# API endpoint for the external bucket. -TF_VAR_external_blob_bucket_endpoint= -TF_VAR_external_blob_bucket_region= -TF_VAR_external_blob_bucket_name= -# Public domain for accessing bucket. -TF_VAR_external_blob_bucket_domain= -TF_VAR_external_blob_bucket_access_key_id= -TF_VAR_external_blob_bucket_secret_access_key= - -# Indexing service configuration ############################################### - -TF_VAR_indexing_service_did=did:web:indexer.storacha.network -TF_VAR_indexing_service_url=https://indexer.storacha.network/claims - -# IPNI configuration ########################################################### - -TF_VAR_ipni_announce_urls=["https://cid.contact/announce"] diff --git a/deploy/Makefile b/deploy/Makefile deleted file mode 100644 index 732c6c00..00000000 --- a/deploy/Makefile +++ /dev/null @@ -1,62 +0,0 @@ -ifneq (,$(wildcard ./.env)) - include .env - export -else - $(error You haven't setup your .env file. Please refer to the readme) -endif -VERSION=$(shell awk -F'"' '/"version":/ {print $$4}' ../version.json) -LAMBDA_GOOS=linux -LAMBDA_GOARCH=arm64 -LAMBDA_GOCC?=go -LAMBDA_GOFLAGS=-tags=lambda.norpc -ldflags="-s -w -X github.com/storacha/piri/pkg/build.version=$(VERSION)" -LAMBDA_CGO_ENABLED=0 -LAMBDAS=build/getclaim/bootstrap build/getroot/bootstrap build/postad/bootstrap build/postroot/bootstrap build/putblob/bootstrap build/publisher/bootstrap build/advertisementpublisher/bootstrap - -.PHONY: clean-lambda - -clean-lambda: - rm -rf build - -.PHONY: clean-terraform - -clean-terraform: - tofu -chdir=app destroy - -.PHONY: clean - -clean: clean-terraform clean-lambda - -lambdas: $(LAMBDAS) - -.PHONY: $(LAMBDAS) - -$(LAMBDAS): build/%/bootstrap: - GOOS=$(LAMBDA_GOOS) GOARCH=$(LAMBDA_GOARCH) CGO_ENABLED=$(LAMBDA_CGO_ENABLED) $(LAMBDA_GOCC) build $(LAMBDA_GOFLAGS) -o $@ ../cmd/lambda/$* - -app/.terraform: - TF_WORKSPACE= tofu -chdir=app init - -.tfworkspace: app/.terraform - TF_WORKSPACE= tofu -chdir=app workspace new $(TF_WORKSPACE) - touch .tfworkspace - -.PHONY: init - -init: app/.terraform .tfworkspace - -.PHONY: validate - -validate: app/.terraform .tfworkspace - tofu -chdir=app validate - -.PHONY: plan - -plan: app/.terraform .tfworkspace $(LAMBDAS) - tofu -chdir=app plan - -.PHONY: apply - -apply: app/.terraform .tfworkspace $(LAMBDAS) - tofu -chdir=app apply - -shared: diff --git a/deploy/README.md b/deploy/README.md deleted file mode 100644 index ecff16d0..00000000 --- a/deploy/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# Deployment - -Deployment of a Storage Node to AWS is managed by Terraform which you can invoke with `make`. - -First, install OpenTofu e.g. - -```sh -brew install opentofu -``` - -...or for Linux distributions that support Snap: - -```sh -snap install --classic opentofu -``` - -...for other Operating Systems see: https://opentofu.org/docs/intro/install - -### AWS settings - -The Terraform configuration will fetch AWS settings (such as credentials and the region to deploy resources to) from your local AWS configuration. Although an installation of the AWS CLI is not strictly required, it can be a convenient way to manage these settings. - -OpenTofu will go to the same places as the AWS CLI to find settings, which means it will read environment variables such as `AWS_REGION` and `AWS_PROFILE` and the `~/.aws/config` and `~/.aws/credentials` files. - -Make sure you are using the correct AWS profile and region before invoking `make` targets. - -### `.env` - -You need to first create a `.env` with relevant vars. Copy `.env.tpl` to `.env`. Set required variables and any optional variables you want to set. Explanations for all variables can be found in the template. - -### Deployment commands - -Note that these commands will call needed prerequisites -- `make apply` will essentially do all of these start to finish. - -#### `make lambdas` - -This will simply compile the lambdas locally and put then in the `build` directory. - -#### `make init` - -You should only need to run this once -- initializes your terraform deployment and workspace. Make sure you've set `TF_WORKSPACE` first! - -If the `make init` fails you will need to execute `tofu init` directly from the `deploy/app` folder to install the required dependencies, and it will update the `.terraform.lock.hcl` file if needed. - -#### `make validate` - -This will validate your terraform configuration -- good to run to check errors in any changes you make to terraform configs. - -#### `make plan` - -This will plan a deployment, but not execute it -- useful to see ahead what changes will happen when you run the next deployment. - -#### `make apply` - -The big kahuna! This will deploy all of your changes, including redeploying lambdas if any of code changes. diff --git a/deploy/app/.terraform.lock.hcl b/deploy/app/.terraform.lock.hcl deleted file mode 100644 index 56dd9a5b..00000000 --- a/deploy/app/.terraform.lock.hcl +++ /dev/null @@ -1,37 +0,0 @@ -# This file is maintained automatically by "tofu init". -# Manual edits may be lost in future updates. - -provider "registry.opentofu.org/hashicorp/archive" { - version = "2.6.0" - hashes = [ - "h1:s1OObC0b95ceQkrAMqL4q6wMYDBWYt8swZbLup+UJXI=", - "zh:046b3ba4223002d1cd1c917e8c21b58a636fcd751073745e3db99beebe254dd8", - "zh:1c1ed2ea0927b491689c3c7d178880cd9902f2a5339da8f46c56279920329a27", - "zh:1f17b47ba1bf18bd7bd30ea35c2ba32eaa23f8d08b3a35126edb31daf6ae10fd", - "zh:4b58aaac88335bb2ca482766e2682514fed78ff8cabe5665b6e5dd7c22ff9c81", - "zh:6c7dd6d4ff061d350fc6eb76866905c47450b8b8c1d2e238aa737afd48b6a267", - "zh:7b376916c5b911a3f887fd296c25ced36d8ba742b8482f1e0f092bf8fb008146", - "zh:8661139125b1ea7b89e0084377863dc820cdcbc433bb9a7c445350480f83b2c2", - "zh:e17c9056f210ec9a8c9cfe8a13ecd09ae59ad0a0197c96589b86eb4f7cf5326d", - "zh:ee15bddc7a596cccd400a762b6dadf1c8889faff7c931ae4b39f2e5404188da1", - "zh:f74355e6588daf88ec210d2967fbf5d22fa18c448d2807b8a7049dc777a2dbcb", - ] -} - -provider "registry.opentofu.org/hashicorp/aws" { - version = "5.75.1" - constraints = ">= 5.73.0" - hashes = [ - "h1:HtN4MOr62Yros/uy02prMtPXdrMO8+LDwaacmmeXT2A=", - "zh:0120ce0a9ae404f8af64ecbc3bfcaf80c6715124e611a0307db2796596b3e729", - "zh:08d7bd9855a9d2c49e7e4220d6e8d4c920cb04fee1b018c16c3148480379466b", - "zh:61b4fcd1642586946263ddaa6aa25649dbfbf532b6176dcf803754990847ed11", - "zh:6353215b21f4f51e6be19676c98700eeb6e62883834d4fbfe42b6054a8cd28b3", - "zh:8a91f55fe010cd7431fe159246891e1291bd52a835b72c1ca4061eef2fa9c666", - "zh:99fda160d4b6abe955ef9cbee5d76954458076d499fa6b4f139652551ed6b484", - "zh:af2f09910c09b38fcfcb6da2a192d28b039e25c1c00b92f79d096128905778ae", - "zh:cb6bb124b80be98584be92a51fe2d0dce8300fcadceac36efe0fa9079d88467f", - "zh:e339ee69a341df9f0630da632d0021a6c6bc99c8a844e1c1899b9d2fc7b93d76", - "zh:e6409bd30979c26299805f63884e6e7b33af4ba39f9cc9f93bd53832f2daf325", - ] -} diff --git a/deploy/app/dynamodb.tf b/deploy/app/dynamodb.tf deleted file mode 100644 index 8c1b23b2..00000000 --- a/deploy/app/dynamodb.tf +++ /dev/null @@ -1,141 +0,0 @@ -resource "aws_dynamodb_table" "metadata" { - name = "${terraform.workspace}-${var.app}-metadata" - billing_mode = "PAY_PER_REQUEST" - - attribute { - name = "provider" - type = "S" - } - - attribute { - name = "contextID" - type = "B" - } - - hash_key = "provider" - range_key = "contextID" - - tags = { - Name = "${terraform.workspace}-${var.app}-metadata" - } - - point_in_time_recovery { - enabled = terraform.workspace == "prod" - } - - deletion_protection_enabled = terraform.workspace == "prod" -} - -resource "aws_dynamodb_table" "chunk_links" { - name = "${terraform.workspace}-${var.app}-chunk-links" - billing_mode = "PAY_PER_REQUEST" - - attribute { - name = "provider" - type = "S" - } - - attribute { - name = "contextID" - type = "B" - } - - hash_key = "provider" - range_key = "contextID" - - tags = { - Name = "${terraform.workspace}-${var.app}-chunk-links" - } - - point_in_time_recovery { - enabled = terraform.workspace == "prod" - } - - deletion_protection_enabled = terraform.workspace == "prod" -} - -resource "aws_dynamodb_table" "ran_link_index" { - name = "${terraform.workspace}-${var.app}-ran-link-index" - billing_mode = "PAY_PER_REQUEST" - - attribute { - name = "ran" - type = "S" - } - - attribute { - name = "link" - type = "S" - } - - hash_key = "ran" - range_key = "link" - - tags = { - Name = "${terraform.workspace}-${var.app}-ran-link-index" - } - - point_in_time_recovery { - enabled = terraform.workspace == "prod" - } - - deletion_protection_enabled = terraform.workspace == "prod" -} - - -resource "aws_dynamodb_table" "allocation_store" { - name = "${terraform.workspace}-${var.app}-allocation-store" - billing_mode = "PAY_PER_REQUEST" - - attribute { - name = "hash" - type = "S" - } - - # note: now contains a space DID not invocation CID - attribute { - name = "cause" - type = "S" - } - - hash_key = "hash" - range_key = "cause" - - tags = { - Name = "${terraform.workspace}-${var.app}-allocation-store" - } - - point_in_time_recovery { - enabled = terraform.workspace == "prod" - } - - deletion_protection_enabled = terraform.workspace == "prod" -} - -resource "aws_dynamodb_table" "acceptance_store" { - name = "${terraform.workspace}-${var.app}-acceptance-store" - billing_mode = "PAY_PER_REQUEST" - - attribute { - name = "hash" - type = "S" - } - - attribute { - name = "space" - type = "S" - } - - hash_key = "hash" - range_key = "space" - - tags = { - Name = "${terraform.workspace}-${var.app}-acceptance-store" - } - - point_in_time_recovery { - enabled = terraform.workspace == "prod" - } - - deletion_protection_enabled = terraform.workspace == "prod" -} diff --git a/deploy/app/gateway.tf b/deploy/app/gateway.tf deleted file mode 100644 index 47734198..00000000 --- a/deploy/app/gateway.tf +++ /dev/null @@ -1,173 +0,0 @@ -locals { - #web_functions = { for k, v in local.functions : k => v if(v.route != "") } - web_functions = { for k, v in local.functions : k => v if(try(v.route,"") != "") } - domain_name = terraform.workspace == "prod" ? "${var.app}.${var.domain}" : "${terraform.workspace}.${var.app}.${var.domain}" -} - -resource "aws_apigatewayv2_api" "api" { - name = "${terraform.workspace}-${var.app}-api" - description = "${terraform.workspace} ${var.app} API Gateway" - protocol_type = "HTTP" -} - -resource "aws_apigatewayv2_route" "routes" { - for_each = local.web_functions - api_id = aws_apigatewayv2_api.api.id - route_key = each.value.route - authorization_type = "NONE" - target = "integrations/${aws_apigatewayv2_integration.integrations[each.key].id}" -} - -resource "aws_apigatewayv2_integration" "integrations" { - for_each = local.web_functions - api_id = aws_apigatewayv2_api.api.id - integration_uri = aws_lambda_function.lambda[each.key].invoke_arn - payload_format_version = "2.0" - integration_type = "AWS_PROXY" - connection_type = "INTERNET" -} - -resource "aws_apigatewayv2_deployment" "deployment" { - depends_on = [aws_apigatewayv2_integration.integrations] - triggers = { - redeployment = sha1(join(",", concat( - [for k, v in local.web_functions : jsonencode(aws_apigatewayv2_route.routes[k])], - [for k, v in local.web_functions : jsonencode(aws_apigatewayv2_integration.integrations[k])]) - )) - } - - api_id = aws_apigatewayv2_api.api.id - description = "${terraform.workspace} ${var.app} API Deployment" - lifecycle { - create_before_destroy = true - } -} - -data "terraform_remote_state" "shared" { - backend = "s3" - config = { - bucket = "${var.owner}-terraform-state" - key = "${var.owner}/${var.app}/shared.tfstate" - region = "us-west-2" - } -} - -resource "aws_acm_certificate" "cert" { - domain_name = local.domain_name - validation_method = "DNS" - - lifecycle { - create_before_destroy = true - } -} - -resource "aws_route53_record" "cert_validation" { - allow_overwrite = true - name = tolist(aws_acm_certificate.cert.domain_validation_options)[0].resource_record_name - type = tolist(aws_acm_certificate.cert.domain_validation_options)[0].resource_record_type - zone_id = data.terraform_remote_state.shared.outputs.primary_zone.zone_id - records = [tolist(aws_acm_certificate.cert.domain_validation_options)[0].resource_record_value] - ttl = 60 -} - -resource "aws_acm_certificate_validation" "cert" { - certificate_arn = aws_acm_certificate.cert.arn - validation_record_fqdns = [aws_route53_record.cert_validation.fqdn] -} - -resource "aws_apigatewayv2_domain_name" "custom_domain" { - domain_name = local.domain_name - - domain_name_configuration { - certificate_arn = aws_acm_certificate_validation.cert.certificate_arn - endpoint_type = "REGIONAL" - security_policy = "TLS_1_2" - } -} - -resource "aws_apigatewayv2_stage" "stage" { - api_id = aws_apigatewayv2_api.api.id - name = "$default" - auto_deploy = true - - access_log_settings { - destination_arn = aws_cloudwatch_log_group.access_logs.arn - format = var.access_logging_log_format - } - - lifecycle { - create_before_destroy = true - } -} - -resource "aws_apigatewayv2_api_mapping" "api_mapping" { - api_id = aws_apigatewayv2_api.api.id - stage = aws_apigatewayv2_stage.stage.id - domain_name = aws_apigatewayv2_domain_name.custom_domain.id -} - -resource "aws_route53_record" "api_gateway" { - zone_id = data.terraform_remote_state.shared.outputs.primary_zone.zone_id - name = aws_apigatewayv2_domain_name.custom_domain.domain_name - type = "A" - - alias { - name = aws_apigatewayv2_domain_name.custom_domain.domain_name_configuration[0].target_domain_name - zone_id = aws_apigatewayv2_domain_name.custom_domain.domain_name_configuration[0].hosted_zone_id - evaluate_target_health = false - } -} - -# logging permissions - -resource "aws_api_gateway_account" "api_gateway_account" { - cloudwatch_role_arn = aws_iam_role.api_gateway_logging_role.arn -} - -data "aws_iam_policy_document" "api_gateway_assume_role_policy" { - statement { - actions = ["sts:AssumeRole"] - - principals { - identifiers = [ - "apigateway.amazonaws.com" - ] - type = "Service" - } - - effect = "Allow" - } -} - -data "aws_iam_policy_document" "api_gateway_logging_policy" { - statement { - effect = "Allow" - actions = [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:DescribeLogGroups", - "logs:DescribeLogStreams", - "logs:PutLogEvents", - "logs:GetLogEvents", - "logs:FilterLogEvents" - ] - resources = [ - "*" - ] - } -} - -resource "aws_iam_role" "api_gateway_logging_role" { - name = "${terraform.workspace}-${var.app}-api-gateway-logging-role" - assume_role_policy = data.aws_iam_policy_document.api_gateway_assume_role_policy.json -} - -resource "aws_iam_role_policy" "api_gateway_logging_role_policy" { - name = "${terraform.workspace}-${var.app}-api-gateway-logging-policy" - role = aws_iam_role.api_gateway_logging_role.name - policy = data.aws_iam_policy_document.api_gateway_logging_policy.json -} - -resource "aws_cloudwatch_log_group" "access_logs" { - name = "/aws/vendedlogs/${var.app}/${terraform.workspace}/api-gateway/default" -} \ No newline at end of file diff --git a/deploy/app/lambda.tf b/deploy/app/lambda.tf deleted file mode 100644 index 916e418b..00000000 --- a/deploy/app/lambda.tf +++ /dev/null @@ -1,302 +0,0 @@ -locals { - functions = { - getclaim = { - name = "GETclaim" - route = "GET /claim/{cid}" - } - getroot = { - name = "GETroot" - route = "GET /" - } - publisher = { - name = "publisher" - } - advertisementpublisher = { - name = "advertisementpublisher" - concurrency = 1 - } - postad = { - name = "POSTad" - route = "POST /ad" - } - postroot = { - name = "POSTroot" - route = "POST /" - } - putblob = { - name = "PUTblob" - route = "PUT /blob/{blob}" - } - } -} - -// zip the binary, as we can use only zip files to AWS lambda -data "archive_file" "function_archive" { - for_each = local.functions - - type = "zip" - source_file = "${path.root}/../build/${each.key}/bootstrap" - output_path = "${path.root}/../build/${each.key}/${each.key}.zip" -} - -# Define functions - -resource "aws_lambda_function" "lambda" { - depends_on = [aws_cloudwatch_log_group.lambda_log_group] - for_each = local.functions - - function_name = "${terraform.workspace}-${var.app}-lambda-${each.value.name}" - handler = "bootstrap" - runtime = "provided.al2023" - architectures = ["arm64"] - role = aws_iam_role.lambda_exec.arn - timeout = try(each.value.timeout, 60) - memory_size = try(each.value.memory_size, 128) - reserved_concurrent_executions = try(each.value.concurrency, -1) - source_code_hash = data.archive_file.function_archive[each.key].output_base64sha256 - filename = data.archive_file.function_archive[each.key].output_path # Path to your Lambda zip files - - environment { - variables = { - SENTRY_DSN = var.sentry_dsn - SENTRY_ENVIRONMENT = var.sentry_environment == "" ? terraform.workspace : var.sentry_environment - CHUNK_LINKS_TABLE_NAME = aws_dynamodb_table.chunk_links.id - METADATA_TABLE_NAME = aws_dynamodb_table.metadata.id - IPNI_STORE_BUCKET_NAME = aws_s3_bucket.ipni_store_bucket.bucket - IPNI_ANNOUNCE_URLS = var.ipni_announce_urls - PRIVATE_KEY = aws_ssm_parameter.private_key.name - PUBLIC_URL = "https://${aws_apigatewayv2_domain_name.custom_domain.domain_name}" - IPNI_STORE_BUCKET_REGIONAL_DOMAIN = aws_s3_bucket.ipni_store_bucket.bucket_regional_domain_name - CLAIM_STORE_BUCKET_NAME = aws_s3_bucket.claim_store_bucket.bucket - ALLOCATIONS_TABLE_NAME = aws_dynamodb_table.allocation_store.id - ACCEPTANCE_TABLE_NAME = aws_dynamodb_table.acceptance_store.id - BLOB_STORE_BUCKET_ENDPOINT = var.use_external_blob_bucket ? var.external_blob_bucket_endpoint : "" - BLOB_STORE_BUCKET_REGION = var.use_external_blob_bucket ? var.external_blob_bucket_region : aws_s3_bucket.blob_store_bucket.region - BLOB_STORE_BUCKET_ACCESS_KEY_ID = var.use_external_blob_bucket ? aws_ssm_parameter.external_blob_bucket_access_key_id[0].name : "" - BLOB_STORE_BUCKET_SECRET_ACCESS_KEY = var.use_external_blob_bucket ? aws_ssm_parameter.external_blob_bucket_secret_access_key[0].name : "" - BLOB_STORE_BUCKET_REGIONAL_DOMAIN = var.use_external_blob_bucket ? var.external_blob_bucket_domain : aws_s3_bucket.blob_store_bucket.bucket_regional_domain_name - BLOB_STORE_BUCKET_NAME = var.use_external_blob_bucket ? var.external_blob_bucket_name : aws_s3_bucket.blob_store_bucket.bucket - BLOB_STORE_BUCKET_KEY_PATTERN = var.blob_bucket_key_pattern - INDEXING_SERVICE_DID = var.indexing_service_did - INDEXING_SERVICE_URL = var.indexing_service_url - INDEXING_SERVICE_PROOF = var.indexing_service_proof - RAN_LINK_INDEX_TABLE_NAME = aws_dynamodb_table.ran_link_index.id - RECEIPT_STORE_BUCKET_NAME = aws_s3_bucket.receipt_store_bucket.id - IPNI_PUBLISHER_QUEUE_ID = aws_sqs_queue.ipni_publisher.id - IPNI_PUBLISHER_BUCKET_NAME = aws_s3_bucket.ipni_publisher.bucket - IPNI_ADVERTISEMENT_PUBLISHING_QUEUE_ID = aws_sqs_queue.ipni_advertisement_publishing.id - PRINCIPAL_MAPPING = var.principal_mapping, - PIRI_NETWORK = var.network, - } - } -} - -# Access for the gateway - -resource "aws_lambda_permission" "api_gateway" { - for_each = aws_lambda_function.lambda - - statement_id = "AllowAPIGatewayInvoke" - action = "lambda:InvokeFunction" - function_name = each.value.function_name - principal = "apigateway.amazonaws.com" - source_arn = "${aws_apigatewayv2_api.api.execution_arn}/*/*" -} - -# Logging - -resource "aws_cloudwatch_log_group" "lambda_log_group" { - for_each = local.functions - name = "/aws/lambda/${terraform.workspace}-${var.app}-lambda-${each.value.name}" - retention_in_days = 7 - lifecycle { - prevent_destroy = false - } -} - -# Role policies and access to resources - -resource "aws_iam_role" "lambda_exec" { - name = "${terraform.workspace}-${var.app}-lambda-exec-role" - - assume_role_policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Action = "sts:AssumeRole" - Effect = "Allow" - Principal = { - Service = "lambda.amazonaws.com" - } - } - ] - }) -} - -data "aws_iam_policy_document" "lambda_dynamodb_put_get_document" { - statement { - actions = [ - "dynamodb:GetItem", - "dynamodb:PutItem", - "dynamodb:Query" - ] - resources = [ - aws_dynamodb_table.chunk_links.arn, - aws_dynamodb_table.metadata.arn, - aws_dynamodb_table.ran_link_index.arn, - aws_dynamodb_table.allocation_store.arn, - aws_dynamodb_table.acceptance_store.arn - ] - } -} - -resource "aws_iam_policy" "lambda_dynamodb_put_get" { - name = "${terraform.workspace}-${var.app}-lambda-dynamodb-put-get" - description = "This policy will be used by the lambda to put and get data from DynamoDB" - policy = data.aws_iam_policy_document.lambda_dynamodb_put_get_document.json -} - -resource "aws_iam_role_policy_attachment" "lambda_dynamodb_put_get" { - role = aws_iam_role.lambda_exec.name - policy_arn = aws_iam_policy.lambda_dynamodb_put_get.arn -} - - -data "aws_iam_policy_document" "lambda_s3_put_get_document" { - statement { - actions = [ - "s3:GetObject", - "s3:PutObject", - "s3:HeadObject", - ] - resources = [ - "${aws_s3_bucket.blob_store_bucket.arn}/*", - "${aws_s3_bucket.ipni_store_bucket.arn}/*", - "${aws_s3_bucket.receipt_store_bucket.arn}/*", - "${aws_s3_bucket.claim_store_bucket.arn}/*", - "${aws_s3_bucket.ipni_publisher.arn}/*", - ] - } - - statement { - actions = [ - "s3:ListBucket", "s3:GetBucketLocation" - ] - resources = [ - aws_s3_bucket.blob_store_bucket.arn, - aws_s3_bucket.ipni_store_bucket.arn, - aws_s3_bucket.receipt_store_bucket.arn, - aws_s3_bucket.claim_store_bucket.arn, - aws_s3_bucket.ipni_publisher.arn, - ] - } -} - -resource "aws_iam_policy" "lambda_s3_put_get" { - name = "${terraform.workspace}-${var.app}-lambda-s3-put-get" - description = "This policy will be used by the lambda to put and get objects from S3" - policy = data.aws_iam_policy_document.lambda_s3_put_get_document.json -} - -resource "aws_iam_role_policy_attachment" "lambda_s3_put_get" { - role = aws_iam_role.lambda_exec.name - policy_arn = aws_iam_policy.lambda_s3_put_get.arn -} - -data "aws_iam_policy_document" "lambda_logs_document" { - statement { - actions = [ - "logs:CreateLogStream", - "logs:PutLogEvents", - ] - resources = [ - "arn:aws:logs:*:*:*" - ] - } -} - -resource "aws_iam_policy" "lambda_logs" { - name = "${terraform.workspace}-${var.app}-lambda-logs" - description = "This policy will be used by the lambda to write logs" - policy = data.aws_iam_policy_document.lambda_logs_document.json -} - -resource "aws_iam_role_policy_attachment" "lambda_logs" { - role = aws_iam_role.lambda_exec.name - policy_arn = aws_iam_policy.lambda_logs.arn -} - -data "aws_iam_policy_document" "lambda_ssm_document" { - statement { - - effect = "Allow" - - actions = [ - "ssm:GetParameters", - ] - - resources = var.use_external_blob_bucket ? [ - aws_ssm_parameter.private_key.arn, - aws_ssm_parameter.external_blob_bucket_access_key_id[0].arn, - aws_ssm_parameter.external_blob_bucket_secret_access_key[0].arn, - ] : [aws_ssm_parameter.private_key.arn] - } -} - -resource "aws_iam_policy" "lambda_ssm" { - name = "${terraform.workspace}-${var.app}-lambda-ssm" - description = "This policy will be used by the lambda to access the parameter store" - policy = data.aws_iam_policy_document.lambda_ssm_document.json -} - -resource "aws_iam_role_policy_attachment" "lambda_ssm" { - role = aws_iam_role.lambda_exec.name - policy_arn = aws_iam_policy.lambda_ssm.arn -} - -data "aws_iam_policy_document" "lambda_sqs_document" { - statement { - - effect = "Allow" - - actions = [ - "sqs:SendMessage*", - "sqs:ReceiveMessage", - "sqs:DeleteMessage", - "sqs:GetQueueAttributes" - ] - - resources = [ - aws_sqs_queue.ipni_publisher.arn, - aws_sqs_queue.ipni_advertisement_publishing.arn - ] - } -} - -resource "aws_iam_policy" "lambda_sqs" { - name = "${terraform.workspace}-${var.app}-lambda-sqs" - description = "This policy will be used by the lambda to send messages to an SQS queue" - policy = data.aws_iam_policy_document.lambda_sqs_document.json -} - -resource "aws_iam_role_policy_attachment" "lambda_sqs" { - role = aws_iam_role.lambda_exec.name - policy_arn = aws_iam_policy.lambda_sqs.arn -} - -# event source mappings - - -resource "aws_lambda_event_source_mapping" "ipni_publisher_source_mapping" { - event_source_arn = aws_sqs_queue.ipni_publisher.arn - enabled = true - function_name = aws_lambda_function.lambda["publisher"].arn - batch_size = terraform.workspace == "prod" ? 10 : 1 -} - -resource "aws_lambda_event_source_mapping" "ipni_advertisement_publishing_source_mapping" { - event_source_arn = aws_sqs_queue.ipni_advertisement_publishing.arn - enabled = true - function_name = aws_lambda_function.lambda["advertisementpublisher"].arn - batch_size = terraform.workspace == "prod" ? 10 : 1 -} diff --git a/deploy/app/main.tf b/deploy/app/main.tf deleted file mode 100644 index 6557fa9c..00000000 --- a/deploy/app/main.tf +++ /dev/null @@ -1,37 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 5.73.0" - } - archive = { - source = "hashicorp/archive" - } - } - backend "s3" { - bucket = "${var.owner}-terraform-state" - key = "${var.owner}/${var.app}/terraform.tfstate" - region = "us-west-2" - } -} - -provider "aws" { - region = var.region - allowed_account_ids = var.allowed_account_ids - default_tags { - - tags = { - "Environment" = terraform.workspace - "ManagedBy" = "OpenTofu" - Owner = "${var.owner}" - Team = "${var.team}" - Organization = "${var.org}" - Project = "${var.app}" - } - } -} - -provider "aws" { - alias = "virginia" - region = "us-east-1" -} \ No newline at end of file diff --git a/deploy/app/s3.tf b/deploy/app/s3.tf deleted file mode 100644 index c9554108..00000000 --- a/deploy/app/s3.tf +++ /dev/null @@ -1,110 +0,0 @@ -resource "aws_s3_bucket" "ipni_store_bucket" { - bucket = "${terraform.workspace}-${var.app}-ipni-store-bucket" -} - -resource "aws_s3_bucket_public_access_block" "ipni_store_bucket" { - bucket = aws_s3_bucket.ipni_store_bucket.id - - block_public_acls = true - block_public_policy = false - ignore_public_acls = true - restrict_public_buckets = false -} - -resource "aws_s3_bucket_cors_configuration" "ipni_store_cors" { - bucket = aws_s3_bucket.ipni_store_bucket.bucket - - cors_rule { - allowed_headers = ["*"] - allowed_methods = ["GET", "HEAD"] - allowed_origins = ["*"] - expose_headers = ["Content-Length", "Content-Type", "Content-MD5", "ETag"] - max_age_seconds = 86400 - } -} - -resource "aws_s3_bucket_policy" "ipni_store_policy" { - depends_on = [aws_s3_bucket_public_access_block.ipni_store_bucket] - bucket = aws_s3_bucket.ipni_store_bucket.id - - policy = jsonencode({ - "Version" : "2012-10-17", - "Statement" : [ - { - "Sid" : "PublicRead", - "Effect" : "Allow", - "Principal" : "*", - "Action" : ["s3:GetObject", "s3:GetObjectVersion"], - "Resource" : ["${aws_s3_bucket.ipni_store_bucket.arn}/*"] - } - ] - }) -} - -resource "aws_s3_bucket" "blob_store_bucket" { - bucket = "${terraform.workspace}-${var.app}-blob-store-bucket" -} - -resource "aws_s3_bucket_public_access_block" "blob_store_bucket" { - bucket = aws_s3_bucket.blob_store_bucket.id - - block_public_acls = false - block_public_policy = false - ignore_public_acls = false - restrict_public_buckets = false -} - -resource "aws_s3_bucket_cors_configuration" "blob_store_cors" { - bucket = aws_s3_bucket.blob_store_bucket.bucket - - cors_rule { - allowed_headers = ["*"] - allowed_methods = ["GET", "HEAD"] - allowed_origins = ["*"] - expose_headers = ["Content-Length", "Content-Type", "Content-MD5", "ETag"] - max_age_seconds = 86400 - } -} - -resource "aws_s3_bucket_policy" "blob_store_policy" { - depends_on = [aws_s3_bucket_public_access_block.blob_store_bucket] - bucket = aws_s3_bucket.blob_store_bucket.id - - policy = jsonencode({ - "Version" : "2012-10-17", - "Statement" : [ - { - "Sid" : "PublicRead", - "Effect" : "Allow", - "Principal" : "*", - "Action" : ["s3:GetObject", "s3:GetObjectVersion"], - "Resource" : ["${aws_s3_bucket.blob_store_bucket.arn}/*"] - } - ] - }) -} - -resource "aws_s3_bucket" "receipt_store_bucket" { - bucket = "${terraform.workspace}-${var.app}-receipt-store-bucket" -} - -resource "aws_s3_bucket" "claim_store_bucket" { - bucket = "${terraform.workspace}-${var.app}-claim-store-bucket" -} - -resource "aws_s3_bucket" "ipni_publisher" { - bucket = "${terraform.workspace}-${var.app}-ipni-publisher" -} - -resource "aws_s3_bucket_lifecycle_configuration" "ipni_publisher_lifecycle" { - bucket = aws_s3_bucket.ipni_publisher.id - - rule { - id = "${terraform.workspace}-${var.app}-ipni-publisher-expire-all-rule" - status = "Enabled" - - expiration { - days = 14 - } - } -} diff --git a/deploy/app/secrets.tf b/deploy/app/secrets.tf deleted file mode 100644 index 471d5dca..00000000 --- a/deploy/app/secrets.tf +++ /dev/null @@ -1,34 +0,0 @@ -resource "aws_ssm_parameter" "private_key" { - name = "/${var.app}/${terraform.workspace}/Secret/PRIVATE_KEY/value" - description = "private key for the deployed environment" - type = "SecureString" - value = var.private_key - - tags = { - environment = "production" - } -} - -resource "aws_ssm_parameter" "external_blob_bucket_access_key_id" { - count = var.use_external_blob_bucket ? 1 : 0 - name = "/${var.app}/${terraform.workspace}/Secret/EXTERNAL_BLOB_BUCKET_ACCESS_KEY_ID/value" - description = "access key ID for an externally hosted blob bucket" - type = "SecureString" - value = var.external_blob_bucket_access_key_id - - tags = { - environment = "production" - } -} - -resource "aws_ssm_parameter" "external_blob_bucket_secret_access_key" { - count = var.use_external_blob_bucket ? 1 : 0 - name = "/${var.app}/${terraform.workspace}/Secret/EXTERNAL_BLOB_BUCKET_SECRET_ACCESS_KEY/value" - description = "secret access key for an externally hosted blob bucket" - type = "SecureString" - value = var.external_blob_bucket_secret_access_key - - tags = { - environment = "production" - } -} diff --git a/deploy/app/sqs.tf b/deploy/app/sqs.tf deleted file mode 100644 index 915c877e..00000000 --- a/deploy/app/sqs.tf +++ /dev/null @@ -1,63 +0,0 @@ -resource "aws_sqs_queue" "ipni_publisher" { - name = "${terraform.workspace}-${var.app}-ipni-publisher.fifo" - fifo_queue = true - content_based_deduplication = true - deduplication_scope = "messageGroup" - fifo_throughput_limit = "perMessageGroupId" - redrive_policy = jsonencode({ - deadLetterTargetArn = aws_sqs_queue.ipni_publisher_deadletter.arn - maxReceiveCount = 4 - }) - tags = { - Name = "${terraform.workspace}-${var.app}-ipni-publisher" - } - visibility_timeout_seconds = 60 -} - -resource "aws_sqs_queue" "ipni_publisher_deadletter" { - fifo_queue = true - content_based_deduplication = true - deduplication_scope = "messageGroup" - fifo_throughput_limit = "perMessageGroupId" - name = "${terraform.workspace}-${var.app}-ipni-publisher-deadletter.fifo" -} - -resource "aws_sqs_queue_redrive_allow_policy" "ipni_publisher" { - queue_url = aws_sqs_queue.ipni_publisher_deadletter.id - - redrive_allow_policy = jsonencode({ - redrivePermission = "byQueue", - sourceQueueArns = [aws_sqs_queue.ipni_publisher.arn] - }) -} - -resource "aws_sqs_queue" "ipni_advertisement_publishing" { - name = "${terraform.workspace}-${var.app}-ipni-advertisement-publishing.fifo" - fifo_queue = true - content_based_deduplication = true - redrive_policy = jsonencode({ - deadLetterTargetArn = aws_sqs_queue.ipni_advertisement_publishing_deadletter.arn - maxReceiveCount = 4 - }) - tags = { - Name = "${terraform.workspace}-${var.app}-ipni-advertisement-publishing" - } - visibility_timeout_seconds = 60 -} - -resource "aws_sqs_queue" "ipni_advertisement_publishing_deadletter" { - fifo_queue = true - content_based_deduplication = true - deduplication_scope = "messageGroup" - fifo_throughput_limit = "perMessageGroupId" - name = "${terraform.workspace}-${var.app}-ipni-advertisement-publishing-deadletter.fifo" -} - -resource "aws_sqs_queue_redrive_allow_policy" "ipni_advertisement_publishing" { - queue_url = aws_sqs_queue.ipni_advertisement_publishing_deadletter.id - - redrive_allow_policy = jsonencode({ - redrivePermission = "byQueue", - sourceQueueArns = [aws_sqs_queue.ipni_advertisement_publishing.arn] - }) -} diff --git a/deploy/app/variables.tf b/deploy/app/variables.tf deleted file mode 100644 index 56a5e6b5..00000000 --- a/deploy/app/variables.tf +++ /dev/null @@ -1,149 +0,0 @@ -variable "app" { - description = "name of the application" - type = string - default = "storage" -} - -variable "owner" { - description = "owner of the resources" - type = string - default = "storacha" -} - -variable "team" { - description = "name of team managing working on the project" - type = string - default = "Storacha Engineering" -} - -variable "org" { - description = "name of the organization managing the project" - type = string - default = "Storacha" -} - -variable "domain" { - description = "domain name to use for the deployment (will be prefixed with app name)" - type = string - default = "storacha.network" -} - -variable "region" { - description = "aws region for all services" - type = string - default = "us-west-2" -} - -variable "allowed_account_ids" { - description = "account IDs used for AWS" - type = list(string) - default = ["0"] -} - -variable "private_key" { - description = "private_key for the peer for this deployment" - type = string -} - -variable "network" { - description = "network the deployment targets" - type = string -} - -variable "indexing_service_did" { - description = "did to use for the indexer" - type = string - default = "did:web:indexer.storacha.network" -} - -variable "indexing_service_url" { - description = "url to use for the indexer" - type = string - default = "https://indexer.storacha.network/claims" -} - -variable "indexing_service_proof" { - description = "UCAN delegation to prove this storage node can access the indexer" - type = string -} - -variable "access_logging_log_format" { - type = string - description = "The log format to use for access logging." - default = "{\"apiId\": \"$context.apiId\", \"requestId\": \"$context.requestId\", \"extendedRequestId\": \"$context.extendedRequestId\", \"httpMethod\": \"$context.httpMethod\", \"path\": \"$context.path\", \"protocol\": \"$context.protocol\", \"requestTime\": \"$context.requestTime\", \"requestTimeEpoch\": \"$context.requestTimeEpoch\", \"status\": $context.status, \"responseLatency\": $context.responseLatency, \"responseLength\": $context.responseLength}" -} - -variable "principal_mapping" { - type = string - description = "JSON encoded mapping of did:web to did:key" - default = "" -} - -variable "blob_bucket_key_pattern" { - type = string - description = "Optional key pattern (with {blob} specifier) for blob bucket" - default = "blob/{blob}" -} - -variable "sentry_dsn" { - type = string - description = "DSN for Sentry (get it from your Sentry project's properties). Leave unset to disable error reporting." - default = "" -} - -variable "sentry_environment" { - type = string - description = "Environment name for Sentry" - default = "" -} - -variable "ipni_announce_urls" { - type = string - description = "Optional JSON array of IPNI node URLs to announce chain updates to." - default = "[\"https://cid.contact\"]" -} - -// Externally hosted, S3 compatible blob bucket? These variables are for you. -// Note: credentials MUST have s3:GetObject, s3:PutObject s3:ListBucket perms. - -variable "use_external_blob_bucket" { - type = bool - description = "Is the blob bucket externally hosted (but S3 compatible)?" - default = false -} - -variable "external_blob_bucket_endpoint" { - type = string - description = "Optional endpoint of an external blob bucket" - default = "" -} - -variable "external_blob_bucket_region" { - type = string - description = "Optional region of an external blob bucket" - default = "" -} - -variable "external_blob_bucket_name" { - type = string - description = "Optional name of an external blob bucket" - default = "" -} - -variable "external_blob_bucket_domain" { - type = string - description = "Optional domain name for the external blob bucket." - default = "" -} - -variable "external_blob_bucket_access_key_id" { - type = string - description = "Optional access key ID for external blob bucket" - default = "" -} - -variable "external_blob_bucket_secret_access_key" { - type = string - description = "Optional secret access key for external blob bucket" - default = "" -} diff --git a/deploy/pdp/README.md b/deploy/pdp/README.md deleted file mode 100644 index 4496a824..00000000 --- a/deploy/pdp/README.md +++ /dev/null @@ -1,137 +0,0 @@ -# PDP Deployment (Lotus + Curio + Yugabyte) - -This Terraform configuration deploys a PDP node using three major components: -1. **Lotus** – Interacts with the Filecoin network. -2. **Curio** – Coordinates storage, PDP (Provable Data Possession) services, and network communication. -3. **Yugabyte** – Provides a distributed database for Curio’s metadata. - -## Quick Start - -1. **Create/Configure Wallets & Keys** - - Ensure you have: - - **BLS Lotus wallet** (for configuring Curio). - - **Delegated Lotus wallet** (for interacting with the PDP smart contract). - - A **PEM file** (`service.pem`) to authenticate a Storacha storage node with the Curio API. - -2. **Create `terraform.tfvars`** - Provide the file paths to your wallet JSON files and PEM key file: - ```hcl - lotus_wallet_bls_file = "./bls.json" - lotus_wallet_delegated_file = "./delegated.json" - curio_service_pem_key_file = "./service.pem" - -3. **Plan & Apply** - ```bash - tofu plan -var-file=terraform.tfvars - tofu apply -var-file=terraform.tfvars - ``` - Once apply completes, note the public IP address in the output and confirm the node is also available at https://pdp.storacha.network. - - - -## Detailed Setup Steps - -There are three requirements to deploy this terraform: -1. A BLS Lotus wallet with funds for configuring Curio (TODO: ideally this isn't required, but for now it is.) -2. A Delegated Lotus wallet with funds for interation with the PDP Smart-Contract. -3. A PEM file containing a public key for authenticating a storacha storage node with the curio API. - -### 1. Create Lotus wallets -**a. Create a BLS wallet:** -```bash -export BLS_WALLET=$(lotus wallet new bls) \ -&& lotus wallet export "$BLS_WALLET" \ -| xxd -r -p \ -| jq -c --arg address "$BLS_WALLET" '.Address = $address' \ -> bls.json -``` - -**b. Create a Delegated wallet:** -```bash -export DELEGATED_WALLET=$(lotus wallet new delegated) \ -&& lotus wallet export "$DELEGATED_WALLET" \ -| xxd -r -p \ -| jq -c --arg address "$DELEGATED_WALLET" '.Address = $address' \ -> delegated.json -``` - -**c. Fund both wallets using the [calibration faucet](https://faucet.calibnet.chainsafe-fil.io/funds.html):** -- BLS wallet address: `cat bls.json | jq '.Address'` -- Delegated wallet address: `cat delegated.json | jq '.Address'` - -### 2. Create a PEM file the storacha storage node -_draw the rest of the !%^&$#@ owl..._ - -**TODO: we may need to add support to the `storage identity` command for this.** - - -You need a PEM file (`service.pem`) so Curio can authorize API requests from the Storacha storage node. - - -Generally this looks something like: -`./storage identity generate > service.pem` - -## 3. Populate `terraform.tfvars` - -Create a file named `terraform.tfvars` with the necessary variables: -```terraform -lotus_wallet_bls_file = "./bls.json" -lotus_wallet_delegated_file = "./delegate.json" -curio_service_pem_key_file = "./service.pem" -``` -You can also customize things like instance size, region, or volume size if needed. - - -## 4. Deploy -a. **Plan:** -``` -tofu plan -var-vars=terraform.tfvars -``` - -b. **Apply:** -``` -tofu apply -var-vars=terraform.tfvars -``` - -When finished, the Terraform output shows your instance’s public IP. You can also reach it at https://pdp.storacha.network. - - -## Monitoring & Next Steps -### 1. Access the Instance - -Use the provided private key from [1Password](https://start.1password.com/open/i?a=SJ2Q5WC77NHLDFRUAUF6HYKKHU&v=rof22gakdxoldc6exdtsqmo5tu&i=walq46vmpwcvxd6mqvhzkxps6e&h=storachainc.1password.com) (different from service.pem) to SSH: -```bash -$ ssh -i /path/to/key.pem ubuntu@pdp.storacha.network -``` -_(Replace `/path/to/key.pem` with the actual file path.)_ - - -### 2. Check Service States -All key services must reach active before the node is fully ready. You can watch them in real time: -```bash -watch -n 2 ' - for s in yugabyte yugabyte-ready lotus-prestart lotus lotus-ready lotus-poststart curio-prestart curio curio-ready curio-poststart - do - echo "$s: $(systemctl is-active $s)" | ccze -A - done -' -``` -Once all show `active`, the node is operational. - -### 3. Review Logs -To view live logs: -```bash -journalctl -u yugabyte -u yugabyte-ready \ - -u lotus-prestart -u lotus -u lotus-ready -u lotus-poststart \ - -u curio-prestart -u curio -u curio-ready -u curio-poststart \ - -f -o short-iso | ccze -A -``` - -### 4. Monitor Resource Usage -Use top to see CPU and memory usage for Lotus and Curio: -```bash -top -p $(pgrep -d',' -f lotus),$(pgrep -d',' -f curio) -``` - -### 5. Next Steps -Optionally Deploy and Connect a Storacha Storage Node to the Curio instance for PDP interactions diff --git a/deploy/pdp/cloud-init.yaml.tpl b/deploy/pdp/cloud-init.yaml.tpl deleted file mode 100644 index 353bd34a..00000000 --- a/deploy/pdp/cloud-init.yaml.tpl +++ /dev/null @@ -1,33 +0,0 @@ -#cloud-config - -package_update: true -packages: - - mesa-opencl-icd - - ocl-icd-opencl-dev - - gcc - - git - - jq - - pkg-config - - curl - - clang - - build-essential - - hwloc - - libhwloc-dev - - wget - - aria2 - - pgcli - - python-is-python3 - - ccze - -write_files: -%{ for wf in write_files ~} - - path: ${wf.path} - permissions: "${wf.permissions}" - encoding: base64 - content: ${base64encode(wf.content)} -%{ endfor } - -runcmd: -%{ for cmd in runcmd_steps ~} - - ${cmd} -%{ endfor } diff --git a/deploy/pdp/configs/curio-pdp.toml b/deploy/pdp/configs/curio-pdp.toml deleted file mode 100644 index 2c75a1ec..00000000 --- a/deploy/pdp/configs/curio-pdp.toml +++ /dev/null @@ -1,10 +0,0 @@ -[HTTP] -DomainName = "${CURIO_DOMAIN_NAME}" -Enable = true -ListenAddress = "0.0.0.0:443" - -[Subsystems] -EnableCommP = true -EnableMoveStorage = true -EnablePDP = true -EnableParkPiece = true diff --git a/deploy/pdp/configs/curio-storage.toml b/deploy/pdp/configs/curio-storage.toml deleted file mode 100644 index de9b1cd4..00000000 --- a/deploy/pdp/configs/curio-storage.toml +++ /dev/null @@ -1,2 +0,0 @@ -[Subsystems] -EnableMoveStorage = true diff --git a/deploy/pdp/main.tf b/deploy/pdp/main.tf deleted file mode 100644 index a52dbea8..00000000 --- a/deploy/pdp/main.tf +++ /dev/null @@ -1,73 +0,0 @@ -resource "aws_security_group" "pdp_node_sg" { - name = "pdp_node_sg" - description = "Security group for PDP Node" - vpc_id = aws_vpc.pdp_vpc.id - - ingress { - description = "SSH" - from_port = 22 - to_port = 22 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - ingress { - description = "HTTP" - from_port = 80 - to_port = 80 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - ingress { - description = "HTTPS" - from_port = 443 - to_port = 443 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - egress { - from_port = 0 - to_port = 0 - protocol = -1 - cidr_blocks = ["0.0.0.0/0"] - } -} - -data "aws_route53_zone" "pdp_zone" { - name = "pdp.storacha.network" -} - -resource "aws_route53_record" "pdp_record" { - zone_id = data.aws_route53_zone.pdp_zone.zone_id - name = "pdp.storacha.network" # Apex record - type = "A" - ttl = 300 - records = [aws_instance.pdp_node.public_ip] -} - -resource "aws_instance" "pdp_node" { - ami = "ami-00c257e12d6828491" #"ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-20250115" - instance_type = var.instance_type - subnet_id = aws_subnet.pdp_public_subnet.id - vpc_security_group_ids = [aws_security_group.pdp_node_sg.id] - - key_name = "forrest-lotus-curio" - - user_data = local.cloud_init - - tags = { - "Environment" = terraform.workspace - "ManagedBy" = "OpenTofu" - Owner = "storacha" - Team = "Storacha Engineering" - Organization = "Storacha" - Project = var.app - } - - root_block_device { - volume_size = var.volume_size - volume_type = var.volume_type - } -} diff --git a/deploy/pdp/output.tf b/deploy/pdp/output.tf deleted file mode 100644 index d62fcf9a..00000000 --- a/deploy/pdp/output.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "instance_public_ip" { - description = "Public IP of the instance" - value = aws_instance.pdp_node.public_ip -} diff --git a/deploy/pdp/provider.tf b/deploy/pdp/provider.tf deleted file mode 100644 index bc0aff76..00000000 --- a/deploy/pdp/provider.tf +++ /dev/null @@ -1,33 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 5.73.0" - } - archive = { - source = "hashicorp/archive" - } - } - backend "s3" { - bucket = "${var.owner}-terraform-state" - key = "${var.owner}/${var.app}/terraform.tfstate" - region = "us-west-2" - } -} - -provider "aws" { - region = var.region - allowed_account_ids = var.allowed_account_ids - default_tags { - - tags = { - "Environment" = terraform.workspace - "ManagedBy" = "OpenTofu" - Owner = "${var.owner}" - Team = "${var.team}" - Organization = "${var.org}" - Project = "${var.app}" - } - } -} - diff --git a/deploy/pdp/scripts/curio-poststart.sh b/deploy/pdp/scripts/curio-poststart.sh deleted file mode 100644 index 06d0051f..00000000 --- a/deploy/pdp/scripts/curio-poststart.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -if [ ! -f /var/lib/curio/storage.json ]; then - curio cli --machine=127.0.0.1:12300 \ - storage attach --init --seal --store /var/lib/curio/piecepark/; -fi \ No newline at end of file diff --git a/deploy/pdp/scripts/install-curio.sh b/deploy/pdp/scripts/install-curio.sh deleted file mode 100644 index de7bd4a6..00000000 --- a/deploy/pdp/scripts/install-curio.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -#set -euxo pipefail - -# Clone & checkout pinned version -#cd /tmp -#git clone https://github.com/filecoin-project/curio.git -#cd curio -#git checkout "${CURIO_VERSION}" - -# Build Lotus for target -#make clean "${CURIO_BUILD_TARGET}" - -# Place the final binary somewhere globally accessible -# The actual compiled binary is typically `curio` in the project root -cd /tmp -#curl -o curio -L "https://bafybeibu66ysikactrnccubi2u2g3wsbvgtp4rqpxzzobdwa62y5lm2vye.ipfs.w3s.link/ipfs/bafybeibu66ysikactrnccubi2u2g3wsbvgtp4rqpxzzobdwa62y5lm2vye/curio" -curl -o curio -L "https://bafybeib2wjsd25tq4kbchjve4wqtqs7zs3qrhtwz2qo3mknoijgctbfjyy.ipfs.w3s.link/ipfs/bafybeib2wjsd25tq4kbchjve4wqtqs7zs3qrhtwz2qo3mknoijgctbfjyy/curio" - -cp curio /usr/local/bin/curio -chmod 755 /usr/local/bin/curio - -# Create a standard repo path for storing chain data -mkdir -p "/var/lib/curio" -chown ubuntu:ubuntu "/var/lib/curio" diff --git a/deploy/pdp/scripts/install-go.sh b/deploy/pdp/scripts/install-go.sh deleted file mode 100644 index f585d826..00000000 --- a/deploy/pdp/scripts/install-go.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -cd /tmp -curl -OL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" - -tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz" - -# really __really__ realllllly wish go wasn't such a baby about its environment variables..... -echo "export PATH=\$PATH:/usr/local/go/bin" >> /etc/profile -echo "export GOPATH=/root/go" >> /etc/profile -echo "export HOME=/root" >> /etc/profile \ No newline at end of file diff --git a/deploy/pdp/scripts/install-lotus.sh b/deploy/pdp/scripts/install-lotus.sh deleted file mode 100644 index b7be2b19..00000000 --- a/deploy/pdp/scripts/install-lotus.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -# Clone & checkout pinned version -#cd /tmp -#git clone https://github.com/filecoin-project/lotus.git -#cd lotus -#git checkout "${LOTUS_VERSION}" - -# Build Lotus for target -#make clean "${LOTUS_BUILD_TARGET}" - -# Place the final binary somewhere globally accessible -# The actual compiled binary is typically `lotus` in the project root -cd /tmp -curl -o lotus -L "https://bafybeiha6lzzyafpnnccq73w7xni2i6otmnllxsliuopbfea2h3uywcnku.ipfs.w3s.link/ipfs/bafybeiha6lzzyafpnnccq73w7xni2i6otmnllxsliuopbfea2h3uywcnku/lotus" -cp lotus /usr/local/bin/lotus -chmod 755 /usr/local/bin/lotus - -# Create a standard repo path for storing chain data -mkdir -p "/var/lib/lotus" -chown ubuntu:ubuntu "/var/lib/lotus" diff --git a/deploy/pdp/scripts/install-rust.sh b/deploy/pdp/scripts/install-rust.sh deleted file mode 100644 index dfec21c5..00000000 --- a/deploy/pdp/scripts/install-rust.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -# Install rustup and the latest stable Rust toolchain -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y - -# Add rustup to path -echo "export PATH=\$PATH:\$HOME/.cargo/bin" >> /etc/profile diff --git a/deploy/pdp/scripts/install-yugabyte.sh b/deploy/pdp/scripts/install-yugabyte.sh deleted file mode 100644 index d11a6e13..00000000 --- a/deploy/pdp/scripts/install-yugabyte.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -cd /opt -mkdir -p yugabyte && cd yugabyte -curl -OL "https://downloads.yugabyte.com/releases/${YUGABYTE_VERSION}/yugabyte-${YUGABYTE_VERSION}-b1-linux-x86_64.tar.gz" - -tar xzf "yugabyte-${YUGABYTE_VERSION}-b1-linux-x86_64.tar.gz" --strip-components=1 -ln -sf "/opt/yugabyte/bin/yugabyted" /usr/local/bin/yugabyted diff --git a/deploy/pdp/scripts/lotus-import-snapshot.sh b/deploy/pdp/scripts/lotus-import-snapshot.sh deleted file mode 100644 index e464aee4..00000000 --- a/deploy/pdp/scripts/lotus-import-snapshot.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -# Create a working directory for snapshots -mkdir -p "/opt/lotus-snapshots" -cd "/opt/lotus-snapshots" - -# Download snapshot with aria2c -# Note: '-o' sets output filename -aria2c -x5 "${LOTUS_SNAPSHOT_URL}" -o snapshot.car.zst - -# Import the snapshot into Lotus -lotus --repo="/var/lib/lotus" daemon --halt-after-import --import-snapshot "/opt/lotus-snapshots/snapshot.car.zst" - -# Clean up snapshot to free disk -rm -f "/opt/lotus-snapshots/snapshot.car.zst" diff --git a/deploy/pdp/scripts/lotus-import-wallets.sh b/deploy/pdp/scripts/lotus-import-wallets.sh deleted file mode 100644 index ac477563..00000000 --- a/deploy/pdp/scripts/lotus-import-wallets.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -# Only import if the file exists and is not empty -if [[ -s "/opt/lotus_wallet_bls.json" ]]; then - echo "Importing BLS wallet: /opt/lotus_wallet_bls.json" - lotus --repo=/var/lib/lotus wallet import --format=json-lotus /opt/lotus_wallet_bls.json -else - echo "No BLS wallet file to import." -fi - -if [[ -s "/opt/lotus_wallet_delegated.json" ]]; then - echo "Importing delegated wallet: /opt/lotus_wallet_delegated.json" - lotus --repo=/var/lib/lotus wallet import --format=json-lotus /opt/lotus_wallet_delegated.json -else - echo "No delegated wallet file to import." -fi \ No newline at end of file diff --git a/deploy/pdp/scripts/service-ready.sh b/deploy/pdp/scripts/service-ready.sh deleted file mode 100644 index 9ebe02c7..00000000 --- a/deploy/pdp/scripts/service-ready.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -# Usage: service-ready.sh HOST PORT [TIMEOUT_SECONDS] -# Example: service-ready.sh 127.0.0.1 12300 60 - -HOST="${1}" -PORT="${2}" -TIMEOUT="${3:-60}" - -for i in $(seq 1 "$TIMEOUT"); do - if nc -z "$HOST" "$PORT" >/dev/null 2>&1; then - exit 0 - fi - sleep 1 -done - -echo "Service at $HOST:$PORT didn't become ready within $TIMEOUT seconds." -exit 1 diff --git a/deploy/pdp/services/README.md b/deploy/pdp/services/README.md deleted file mode 100644 index fbcc5eb2..00000000 --- a/deploy/pdp/services/README.md +++ /dev/null @@ -1,88 +0,0 @@ -## Service Architecture -Below is a high-level explanation of how these services are ordered and how they depend on each other. Each service has a distinct role in bringing up a fully functional PDP node using Lotus, Curio, and YugabyteDB. - -## Yugabyte -### 1. `yugabyte.service` -- Starts the single-node YugabyteDB database (yugabyted). -- Depends on `network-online.target` to ensure the instance’s network is ready. -- Runs continuously (Type=simple). -- If it crashes, systemd restarts it. - -### 2. `yugabyte-ready.service` -- One-time check that polls port 5433 to confirm YugabyteDB is actually up and listening. -- Depends on `yugabyte.service` and wants it to be started first. -- Only exits successfully when YugabyteDB is fully ready. - -## Lotus -### 1. `lotus-prestart.service` -- One-time setup that imports a snapshot into Lotus. -- Runs before `lotus.service` and depends only on a functioning network. -- After it completes, Lotus can start its daemon with up-to-date chain data. -- Remains in “`active` (exited)” after successful completion. - -### 2. `lotus.service` -- The main Lotus daemon (Type=simple). -- Depends on `lotus-prestart.service` finishing first (via After/Wants). -- Restarts automatically if it stops unexpectedly. - -### 3. `lotus-ready.service` -- One-time check that polls port 1234 to confirm Lotus is listening and ready. -- Depends on `lotus.service`, ensuring it’s already started. -- Signals “Lotus is ready” once it exits successfully. -- Remains in “`active` (exited)” after successful completion. - -### 4. `lotus-poststart.service` -- One-time post-start task to import wallets. -- Runs only after Lotus is ready, depending on `lotus-ready.service`. -- Remains in “`active` (exited)” after successful completion. - -## Curio -### 1. `curio-prestart.service` -- One-time initialization that configures Curio: -- Ensures Lotus and Yugabyte are ready (depends on `lotus-poststart.service` and `yugabyte-ready.service`). -- Performs commands like curio doit to set up PDP, attach keys, and import configurations. -- This must complete before the main Curio daemon starts. - -### 2. `curio.service` -- The main Curio daemon (Type=simple). -- Depends on `curio-prestart.service` finishing successfully. -- Runs continuously once started. - -### 3. `curio-ready.service` -- One-time check that polls port 12300 to confirm the Curio daemon is fully up. -- Depends on `curio.service` and only succeeds once Curio is actually listening. - -### 4. `curio-poststart.service` -- Post-start tasks (e.g., attaching storage). -- runs after Curio is marked ready. -- Also remains in “`active` (exited)” after successful completion. - -# How They Tie Together -Once all “ready” and “poststart” services exit successfully, you have a fully functional PDP node running Curio (backed by Lotus and Yugabyte), each service restarting automatically if needed. -## 1. Network & Initial Setup -- `network-online.target` becomes `active`, allowing services to begin. -- `yugabyte.service` and `lotus-prestart.service` start independently. - -## 2. Database and Lotus Daemon -- Yugabyte is brought online. -- Lotus imports its snapshot `(lotus-prestart.service`) and then starts the daemon `(lotus.service`). - -## 3. Readiness Checks -- `yugabyte-ready.service` confirms Yugabyte is listening. -- `lotus-ready.service` confirms Lotus is ready. - -## 4. Curio Initialization -- With Lotus and Yugabyte ready, `curio-prestart.service` runs to apply Curio’s initial setup (curio doit, PDP configurations, etc.). -- `curio.service` starts to run the daemon. - -## 5. Final Checks & Post-Start -- `curio-ready.service` confirms Curio is accepting requests on 12300. -- `curio-poststart.service` may do final tasks such as attaching storage or sealing. - - - - - - - - diff --git a/deploy/pdp/services/curio-poststart.service b/deploy/pdp/services/curio-poststart.service deleted file mode 100644 index 98eb4fab..00000000 --- a/deploy/pdp/services/curio-poststart.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Curio Post-Start Tasks -After=curio-ready.service -Wants=curio-ready.service - -[Service] -Type=oneshot -RemainAfterExit=true -ExecStart=/usr/local/bin/curio-poststart.sh - -[Install] -WantedBy=multi-user.target diff --git a/deploy/pdp/services/curio-prestart.service b/deploy/pdp/services/curio-prestart.service deleted file mode 100644 index f83e043b..00000000 --- a/deploy/pdp/services/curio-prestart.service +++ /dev/null @@ -1,39 +0,0 @@ -[Unit] -Description=Curio One-Time Setup -After=network-online.target lotus-poststart.service yugabyte-ready.service -Wants=lotus-poststart.service yugabyte-ready.service - - -[Service] -# We want to run this only once and leave it in "active (exited)" state -Type=oneshot -RemainAfterExit=true - -User=root -Group=root - -EnvironmentFile=/run/curio-address.env - -ExecStartPre=/usr/bin/timeout 30 /bin/sh -c 'until nc -z 127.0.0.1 5433; do sleep 1; done' -# TODO this is a temporary command not in the main branch -# Context: https://filecoinproject.slack.com/archives/C0717TGU7V2/p1740779341321999 -# The one-time command to run: -ExecStart=/usr/bin/env curio doit \ - --repo=/var/lib/lotus/ \ - --sector-size="32 GB" \ - --owner-address=${ADDRESS} \ - --worker-address=${ADDRESS} \ - --sender-address=${ADDRESS} \ - --harmony-hosts=127.0.0.1 \ - --harmony-username=yugabyte \ - --harmony-password=yugabyte \ - --harmony-database=yugabyte \ - --harmony-port=5433 \ - --env-file=/opt/curio.env \ - --add-pdp-service=storacha,/opt/service.pem \ - --import-pdp-key-file=/opt/lotus_wallet_delegated.json -ExecStart=/usr/bin/env curio config set --title=pdp /opt/curio-pdp.toml -ExecStart=/usr/bin/env curio config set --title=storage /opt/curio-storage.toml - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/deploy/pdp/services/curio-ready.service b/deploy/pdp/services/curio-ready.service deleted file mode 100644 index 794bafdd..00000000 --- a/deploy/pdp/services/curio-ready.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Poll for Curio to be ready -After=curio.service -Wants=curio.service - -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/usr/local/bin/service-ready.sh 127.0.0.1 12300 60 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/pdp/services/curio.service b/deploy/pdp/services/curio.service deleted file mode 100644 index 2839d70c..00000000 --- a/deploy/pdp/services/curio.service +++ /dev/null @@ -1,27 +0,0 @@ -[Unit] -Description=Curio -After=network.target curio-prestart.service -Wants=curio-prestart.service - -[Service] -User=root -Group=root -Type=simple - -ExecStartPre=/bin/mkdir -p /var/log/curio - -EnvironmentFile=/opt/curio.env -Environment=LOTUS_PATH=/var/lib/lotus -Environment=CURIO_REPO_PATH=/var/lib/curio -ExecStart=/usr/local/bin/curio run --layers=gui,pdp - -StandardOutput=journal -StandardError=journal - -Restart=always -RestartSec=10 - -LimitNOFILE=65535 - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/deploy/pdp/services/lotus-poststart.service b/deploy/pdp/services/lotus-poststart.service deleted file mode 100644 index 2d8c77fd..00000000 --- a/deploy/pdp/services/lotus-poststart.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Lotus Post-Start Tasks -After=lotus-ready.service -Wants=lotus-ready.service - -[Service] -Type=oneshot -RemainAfterExit=true -ExecStart=/usr/local/bin/lotus-import-wallets.sh - -[Install] -WantedBy=multi-user.target diff --git a/deploy/pdp/services/lotus-prestart.service b/deploy/pdp/services/lotus-prestart.service deleted file mode 100644 index c6d6420a..00000000 --- a/deploy/pdp/services/lotus-prestart.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Lotus One-Time Setup -After=network-online.target - -[Service] -Type=oneshot -RemainAfterExit=true - -User=root -Group=root - -ExecStart=/usr/local/bin/lotus-import-snapshot.sh - -[Install] -WantedBy=multi-user.target diff --git a/deploy/pdp/services/lotus-ready.service b/deploy/pdp/services/lotus-ready.service deleted file mode 100644 index 978f7390..00000000 --- a/deploy/pdp/services/lotus-ready.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Poll for Lotus to be ready -After=lotus.service -Wants=lotus.service - -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/usr/local/bin/service-ready.sh 127.0.0.1 1234 60 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/pdp/services/lotus.service b/deploy/pdp/services/lotus.service deleted file mode 100644 index eb85eeda..00000000 --- a/deploy/pdp/services/lotus.service +++ /dev/null @@ -1,22 +0,0 @@ -[Unit] -Description=Lotus Daemon -After=lotus-prestart.service -Wants=lotus-prestart.service - -[Service] -User=root -Group=root -Type=simple - -ExecStart=/usr/local/bin/lotus --repo /var/lib/lotus daemon - -StandardOutput=journal -StandardError=journal - -Restart=always -RestartSec=10 - -LimitNOFILE=65535 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/pdp/services/yugabyte-ready.service b/deploy/pdp/services/yugabyte-ready.service deleted file mode 100644 index ee353832..00000000 --- a/deploy/pdp/services/yugabyte-ready.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Poll for Yugabyte to be ready -After=yugabyte.service -Wants=yugabyte.service - -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/usr/local/bin/service-ready.sh 127.0.0.1 5433 60 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/pdp/services/yugabyte.service b/deploy/pdp/services/yugabyte.service deleted file mode 100644 index d21ca0f7..00000000 --- a/deploy/pdp/services/yugabyte.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=YugabyteDB single node via yugabyted -After=network-online.target - -[Service] -Type=simple -ExecStart=/usr/local/bin/yugabyted start \ - --advertise_address 127.0.0.1 \ - --master_flags "rpc_bind_addresses=127.0.0.1" \ - --tserver_flags "rpc_bind_addresses=127.0.0.1" \ - --background=false -ExecStop=/opt/yugabyte/bin/yugabyted stop -Restart=on-failure -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/pdp/template.tf b/deploy/pdp/template.tf deleted file mode 100644 index 0fab29ac..00000000 --- a/deploy/pdp/template.tf +++ /dev/null @@ -1,264 +0,0 @@ -/* - This block defines the file structure and service setup for deploying multiple components (e.g., Lotus, Curio, and Yugabyte). - - global_files: Files always installed, regardless of service (e.g., environment scripts, wallet files). - - services: Per-service definitions that specify: - 1. script_files: Shell scripts with optional template variables. - 2. service_files: Systemd .service units for each component. - 3. config_files: Configuration TOML or similar files for the service. - - file_categories: Categorizes file types to determine how they're handled and where they're found on disk. - - write_files: Dynamically builds a list of files to write based on the data in `services` plus any global files. - - all_service_filenames: A flattened list of all .service files used for enabling and starting the services at once. - - systemd_enable_cmd / systemd_start_cmd: Single commands to enable/start all services, letting systemd manage dependency order. - - runcmd_steps: Additional commands to run on instance startup (kernel tweaks, installing each component, etc.), then systemd reload and service enable/start. - - cloud_init: Final rendered user data (writes files, runs commands) for the cloud-init process. - - Integration with cloud-init: - - The `cloud_init` variable is a rendered template that follows the standard cloud-config format (see `cloud-init.yaml.tpl`). - - The lists `write_files` and `runcmd_steps` are inserted into that template, forming the final user-data passed to the instance at boot. - - Cloud-init processes `write_files` to place scripts/configs onto the system with specified permissions, and then executes `runcmd_steps` in order. - - To add a new service: - 1. Create a key under `services` (e.g., "myservice") with `script_files`, `service_files`, and optionally `config_files`. - 2. Place actual scripts in the specified directory (scripts/) and service units in services/ if needed. - 3. Make sure to fill out `filename`, `target_path`, `permissions`, and (if needed) `is_template`/`vars`. - 4. Terraform automatically picks them up in `write_files` and will install/enable them on the instance. -*/ - -locals { - scripts_dir = "${path.module}/scripts" - services_dir = "${path.module}/services" - configs_dir = "${path.module}/configs" - - global_files = [ - { - path = "/usr/local/bin/install_rust.sh" - permissions = "0755" - content = file("${local.scripts_dir}/install-rust.sh") - }, - { - path = "/usr/local/bin/install_go.sh" - permissions = "0755" - content = templatefile("${local.scripts_dir}/install-go.sh", { - GO_VERSION = var.go_version - }) - }, - { - path = "/usr/local/bin/service-ready.sh" - permissions = "0755" - content = file("${local.scripts_dir}/service-ready.sh") - }, - { - path = "/opt/lotus_wallet_bls.json" - permissions = "0400" - content = file(var.lotus_wallet_bls_file) - }, - { - path = "/opt/lotus_wallet_delegated.json" - permissions = "0400" - content = file(var.lotus_wallet_delegated_file) - }, - { - path = "/opt/service.pem" - permissions = "0400" - content = file(var.curio_service_pem_key_file) - } - ] - - services = { - "lotus" = { - script_files = [ - { - filename = "install-lotus.sh" - target_path = "/usr/local/bin/install-lotus.sh" - permissions = "0755" - is_template = true - vars = { - LOTUS_VERSION = var.lotus_version - LOTUS_BUILD_TARGET = var.filecoin_network - } - }, - { - filename = "lotus-import-snapshot.sh" - target_path = "/usr/local/bin/lotus-import-snapshot.sh" - permissions = "0755" - is_template = true - vars = { - LOTUS_SNAPSHOT_URL = var.lotus_snapshot_url - } - }, - { - filename = "lotus-import-wallets.sh" - target_path = "/usr/local/bin/lotus-import-wallets.sh" - permissions = "0755" - is_template = false - vars = {} - } - ] - service_files = [ - { - filename = "lotus.service" - target_path = "/etc/systemd/system/lotus.service" - permissions = "0644" - }, - { - filename = "lotus-prestart.service" - target_path = "/etc/systemd/system/lotus-prestart.service" - permissions = "0644" - }, - { - filename = "lotus-ready.service" - target_path = "/etc/systemd/system/lotus-ready.service" - permissions = "0644" - }, - { - filename = "lotus-poststart.service" - target_path = "/etc/systemd/system/lotus-poststart.service" - permissions = "0644" - } - ] - } - "curio" = { - script_files = [ - { - filename = "install-curio.sh" - target_path = "/usr/local/bin/install-curio.sh" - permissions = "0755" - is_template = true - vars = { - CURIO_VERSION = var.curio_version - CURIO_BUILD_TARGET = var.filecoin_network - } - }, - { - filename = "curio-poststart.sh" - target_path = "/usr/local/bin/curio-poststart.sh" - permissions = "0755" - is_template = false - vars = {} - } - ] - service_files = [ - { - filename = "curio.service" - target_path = "/etc/systemd/system/curio.service" - permissions = "0644" - }, - { - filename = "curio-prestart.service" - target_path = "/etc/systemd/system/curio-prestart.service" - permissions = "0644" - }, - { - filename = "curio-ready.service" - target_path = "/etc/systemd/system/curio-ready.service" - permissions = "0644" - }, - { - filename = "curio-poststart.service" - target_path = "/etc/systemd/system/curio-poststart.service" - permissions = "0644" - } - ] - config_files = [ - { - filename = "curio-pdp.toml" - target_path = "/opt/curio-pdp.toml" - permissions = "0400" - is_template = true - vars = { - CURIO_DOMAIN_NAME = "${var.app}.${var.domain}" - } - }, - { - filename = "curio-storage.toml" - target_path = "/opt/curio-storage.toml" - permissions = "0400" - is_template = false - vars = {} - }, - ] - } - "yugabyte" = { - script_files = [ - { - filename = "install-yugabyte.sh" - target_path = "/usr/local/bin/install-yugabyte.sh" - permissions = "0755" - is_template = true - vars = { - YUGABYTE_VERSION = var.yugabyte_version - } - } - ] - service_files = [ - { - filename = "yugabyte.service" - target_path = "/etc/systemd/system/yugabyte.service" - permissions = "0644" - }, - { - filename = "yugabyte-ready.service" - target_path = "/etc/systemd/system/yugabyte-ready.service" - permissions = "0644" - } - ] - } - } - - file_categories = [ - { key = "script_files", base_dir = local.scripts_dir, default_template = false }, - { key = "service_files", base_dir = local.services_dir, default_template = false }, - { key = "config_files", base_dir = local.configs_dir, default_template = false }, - ] - - write_files = flatten([ - for svc_name, svc_def in local.services : flatten([ - for cat in local.file_categories : ( - try(svc_def[cat.key], []) != [] ? [for f in svc_def[cat.key] : { - path = f.target_path - permissions = f.permissions - content = lookup(f, "is_template", cat.default_template) ? templatefile("${cat.base_dir}/${f.filename}", f.vars) : file("${cat.base_dir}/${f.filename}") - }] : [] - ) - ]) - ]) - - # Flatten all .service filenames into a single list - all_service_filenames = flatten([ - for svc_name, svc_def in local.services : [ - for uf in svc_def.service_files : basename(uf.filename) - ] - ]) - - # build one string for systemctl enable and one for systemctl start by joining all the service filenames in a single line. - # enabling and starting everything with a single command lets systemd manage the order in which services start based - # on their dependencies: `After` and `Want` units - systemd_enable_cmd = "systemctl enable ${join(" ", local.all_service_filenames)}" - systemd_start_cmd = "systemctl start ${join(" ", local.all_service_filenames)}" - - - runcmd_steps = flatten([ - [ - # NB: https://docs.curiostorage.org/installation#system-configuration - ["sysctl -w net.core.rmem_max=2097152"], - ["sysctl -w net.core.rmem_default=2097152"], - ["/usr/local/bin/install-lotus.sh"], - ["/usr/local/bin/install-curio.sh"], - ["/usr/local/bin/install-yugabyte.sh" ], - # NB: required by curio-prestart.service - ["echo ADDRESS=$(cat /opt/lotus_wallet_bls.json | jq -r .Address) > /run/curio-address.env"], - ["systemctl daemon-reload"] - ], - local.systemd_enable_cmd, - local.systemd_start_cmd, - ]) - - # Merge global files with all per-service files - all_write_files = concat(local.global_files, local.write_files) - - # Build our final user data by passing write_files into the template - cloud_init = templatefile("${path.module}/cloud-init.yaml.tpl", { - write_files = local.all_write_files - runcmd_steps = local.runcmd_steps - }) -} \ No newline at end of file diff --git a/deploy/pdp/variables.tf b/deploy/pdp/variables.tf deleted file mode 100644 index 679d2e43..00000000 --- a/deploy/pdp/variables.tf +++ /dev/null @@ -1,101 +0,0 @@ -variable "app" { - description = "name of the application" - type = string - default = "pdp" -} - -variable "owner" { - description = "owner of the resources" - type = string - default = "storacha" -} - -variable "team" { - description = "name of team managing working on the project" - type = string - default = "Storacha Engineering" -} - -variable "org" { - description = "name of the organization managing the project" - type = string - default = "Storacha" -} - -variable "region" { - description = "aws region for all services" - type = string - default = "us-west-2" -} - -variable "domain" { - description = "domain name to use for the deployment (will be prefixed with app name)" - type = string - default = "storacha.network" -} - -variable "allowed_account_ids" { - description = "account IDs used for AWS" - type = list(string) - default = ["505595374361"] -} - -variable "instance_type" { - type = string - default = "t2.2xlarge" -} - -variable "volume_type" { - type = string - default = "gp3" -} - -variable "volume_size" { - type = number - default = 256 -} - -variable "go_version" { - type = string - default = "1.24.4" -} - -variable "yugabyte_version" { - type = string - default = "2.21.0.1" -} - -variable "lotus_version" { - type = string - default = "v1.31.1" -} - -variable "filecoin_network" { - type = string - default = "calibnet" -} - -variable "lotus_snapshot_url" { - type = string - default = "https://forest-archive.chainsafe.dev/latest/calibnet/" -} - -variable "curio_version" { - type = string - default = "feat/pdp" -} - -variable "lotus_wallet_bls_file" { - description = "Path on the local machine (running Terraform) to the lotus BLS wallet file" - type = string -} - -variable "lotus_wallet_delegated_file" { - description = "Path on the local machine (running Terraform) to the lotus delegated wallet file" - type = string -} - -variable "curio_service_pem_key_file" { - description = "Path on the local machine (running Terraform) to the pem key used to authorize api calls to curio" - type = string -} diff --git a/deploy/pdp/vpc.tf b/deploy/pdp/vpc.tf deleted file mode 100644 index 48e7f93b..00000000 --- a/deploy/pdp/vpc.tf +++ /dev/null @@ -1,46 +0,0 @@ -resource "aws_vpc" "pdp_vpc" { - cidr_block = "10.0.0.0/16" - tags = { - Name = "${var.app}-vpc" - } -} - -# Internet Gateway -resource "aws_internet_gateway" "pdp_igw" { - vpc_id = aws_vpc.pdp_vpc.id - tags = { - Name = "${var.app}-igw" - } -} - -# Public Subnet -resource "aws_subnet" "pdp_public_subnet" { - vpc_id = aws_vpc.pdp_vpc.id - cidr_block = "10.0.1.0/24" - availability_zone = "us-west-2a" - map_public_ip_on_launch = true - tags = { - Name = "${var.app}-public-subnet" - } -} - -# Route Table -resource "aws_route_table" "pdp_public_rt" { - vpc_id = aws_vpc.pdp_vpc.id - tags = { - Name = "${var.app}-public-rt" - } -} - -# Default route to the Internet -resource "aws_route" "pdp_public_route_igw" { - route_table_id = aws_route_table.pdp_public_rt.id - destination_cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.pdp_igw.id -} - -# Associate public subnet with the route table -resource "aws_route_table_association" "pdp_public_rta" { - subnet_id = aws_subnet.pdp_public_subnet.id - route_table_id = aws_route_table.pdp_public_rt.id -} diff --git a/deploy/shared/.terraform.lock.hcl b/deploy/shared/.terraform.lock.hcl deleted file mode 100644 index a3f09966..00000000 --- a/deploy/shared/.terraform.lock.hcl +++ /dev/null @@ -1,20 +0,0 @@ -# This file is maintained automatically by "tofu init". -# Manual edits may be lost in future updates. - -provider "registry.opentofu.org/hashicorp/aws" { - version = "5.75.1" - constraints = ">= 5.73.0" - hashes = [ - "h1:HtN4MOr62Yros/uy02prMtPXdrMO8+LDwaacmmeXT2A=", - "zh:0120ce0a9ae404f8af64ecbc3bfcaf80c6715124e611a0307db2796596b3e729", - "zh:08d7bd9855a9d2c49e7e4220d6e8d4c920cb04fee1b018c16c3148480379466b", - "zh:61b4fcd1642586946263ddaa6aa25649dbfbf532b6176dcf803754990847ed11", - "zh:6353215b21f4f51e6be19676c98700eeb6e62883834d4fbfe42b6054a8cd28b3", - "zh:8a91f55fe010cd7431fe159246891e1291bd52a835b72c1ca4061eef2fa9c666", - "zh:99fda160d4b6abe955ef9cbee5d76954458076d499fa6b4f139652551ed6b484", - "zh:af2f09910c09b38fcfcb6da2a192d28b039e25c1c00b92f79d096128905778ae", - "zh:cb6bb124b80be98584be92a51fe2d0dce8300fcadceac36efe0fa9079d88467f", - "zh:e339ee69a341df9f0630da632d0021a6c6bc99c8a844e1c1899b9d2fc7b93d76", - "zh:e6409bd30979c26299805f63884e6e7b33af4ba39f9cc9f93bd53832f2daf325", - ] -} diff --git a/deploy/shared/main.tf b/deploy/shared/main.tf deleted file mode 100644 index 667369e8..00000000 --- a/deploy/shared/main.tf +++ /dev/null @@ -1,37 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 5.73.0" - } - } - backend "s3" { - bucket = "${var.owner}-terraform-state" - key = "${var.owner}/${var.app}/shared.tfstate" - region = "us-west-2" - } -} - -provider "aws" { - region = var.region - allowed_account_ids = var.allowed_account_ids - default_tags { - - tags = { - "Environment" = terraform.workspace - "ManagedBy" = "OpenTofu" - Owner = "${var.owner}" - Team = "${var.team}" - Organization = "${var.org}" - Project = "${var.app}" - } - } -} - -resource "aws_route53_zone" "primary" { - name = "${var.app}.${var.domain}" -} - -output "primary_zone" { - value = aws_route53_zone.primary -} \ No newline at end of file diff --git a/deploy/shared/variables.tf b/deploy/shared/variables.tf deleted file mode 100644 index 52ad6c09..00000000 --- a/deploy/shared/variables.tf +++ /dev/null @@ -1,41 +0,0 @@ -variable "app" { - description = "The name of the application" - type = string - default = "piri" -} - -variable "owner" { - description = "owner of the resources" - type = string - default = "storacha" -} - -variable "team" { - description = "name of team managing working on the project" - type = string - default = "Storacha Engineer" -} - -variable "org" { - description = "name of the organization managing the project" - type = string - default = "Storacha" -} - -variable "region" { - description = "aws region for all services" - type = string - default = "us-west-2" -} - -variable "domain" { - description = "domain name to use for the deployment (will be prefixed with app name)" - type = string - default = "storacha.network" -} - -variable "allowed_account_ids" { - description = "account IDs used for AWS" - type = list(string) - default = ["0"] -} diff --git a/go.mod b/go.mod index b6e0c7a8..291f3c0a 100644 --- a/go.mod +++ b/go.mod @@ -4,17 +4,8 @@ go 1.25.3 require ( github.com/BurntSushi/toml v1.4.0 - github.com/aws/aws-lambda-go v1.47.0 github.com/aws/aws-sdk-go-v2 v1.39.2 - github.com/aws/aws-sdk-go-v2/config v1.31.2 - github.com/aws/aws-sdk-go-v2/credentials v1.18.6 - github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.15.15 - github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.7.50 - github.com/aws/aws-sdk-go-v2/service/dynamodb v1.49.1 github.com/aws/aws-sdk-go-v2/service/s3 v1.88.4 - github.com/aws/aws-sdk-go-v2/service/ssm v1.55.5 - github.com/aws/smithy-go v1.23.0 - github.com/awslabs/aws-lambda-go-api-proxy v0.16.2 github.com/cenkalti/backoff/v5 v5.0.3 github.com/charmbracelet/bubbles v0.21.0 github.com/charmbracelet/bubbletea v1.3.10 @@ -31,7 +22,6 @@ require ( github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 github.com/filecoin-project/go-state-types v0.16.0-rc1 github.com/filecoin-project/lotus v1.32.0-rc1 - github.com/getsentry/sentry-go v0.35.1 github.com/glebarez/go-sqlite v1.21.2 github.com/glebarez/sqlite v1.11.0 github.com/go-playground/validator/v10 v10.14.0 @@ -113,21 +103,14 @@ require ( github.com/StackExchange/wmi v1.2.1 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.9 // indirect - github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.24.5 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.9 // indirect - github.com/aws/aws-sdk-go-v2/service/sqs v1.42.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 // indirect + github.com/aws/smithy-go v1.23.0 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/benbjohnson/clock v1.3.5 // indirect @@ -181,6 +164,7 @@ require ( github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.4 // indirect github.com/gbrlsnchs/jwt/v3 v3.0.1 // indirect + github.com/getsentry/sentry-go v0.35.1 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect @@ -228,7 +212,6 @@ require ( github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.2.4 // indirect @@ -267,6 +250,7 @@ require ( github.com/multiformats/go-varint v0.0.7 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/julianday v1.0.0 // indirect + github.com/onsi/ginkgo v1.16.5 // indirect github.com/onsi/gomega v1.37.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect diff --git a/go.sum b/go.sum index 78832e80..303dfabf 100644 --- a/go.sum +++ b/go.sum @@ -318,60 +318,28 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= -github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI= -github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= github.com/aws/aws-sdk-go-v2 v1.39.2 h1:EJLg8IdbzgeD7xgvZ+I8M1e0fL0ptn/M47lianzth0I= github.com/aws/aws-sdk-go-v2 v1.39.2/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 h1:i8p8P4diljCr60PpJp6qZXNlgX4m2yQFpYk+9ZT+J4E= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1/go.mod h1:ddqbooRZYNoJ2dsTwOty16rM+/Aqmk/GOXrK8cg7V00= -github.com/aws/aws-sdk-go-v2/config v1.31.2 h1:NOaSZpVGEH2Np/c1toSeW0jooNl+9ALmsUTZ8YvkJR0= -github.com/aws/aws-sdk-go-v2/config v1.31.2/go.mod h1:17ft42Yb2lF6OigqSYiDAiUcX4RIkEMY6XxEMJsrAes= -github.com/aws/aws-sdk-go-v2/credentials v1.18.6 h1:AmmvNEYrru7sYNJnp3pf57lGbiarX4T9qU/6AZ9SucU= -github.com/aws/aws-sdk-go-v2/credentials v1.18.6/go.mod h1:/jdQkh1iVPa01xndfECInp1v1Wnp70v3K4MvtlLGVEc= -github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.15.15 h1:2HXPu4MCUKVA/hU0g2DWtYgXjVPsj7Ujd+xif/Yl2fc= -github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.15.15/go.mod h1:fqQI+CG2FX4yVDJORf6QAKLRw16yO+JcB6io1iubcm0= -github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.7.50 h1:SjyghAoNXXDMUUdx4BBFjqyuvuw2DuobVxBBXknsi4A= -github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.7.50/go.mod h1:z4QntVMcpu4UnoKENJl8pFohHHf55MG8kM2fkA4x8fg= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 h1:lpdMwTzmuDLkgW7086jE94HweHCqG+uOJwHf3LZs7T0= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4/go.mod h1:9xzb8/SV62W6gHQGC/8rrvgNXU6ZoYM3sAIJCIrXJxY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9 h1:se2vOWGD3dWQUtfn4wEjRQJb1HK1XsNIt825gskZ970= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9/go.mod h1:hijCGH2VfbZQxqCDN7bwz/4dzxV+hkyhjawAtdPWKZA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9 h1:6RBnKZLkJM4hQ+kN6E7yWFveOTg8NLPHAkqrs4ZPlTU= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9/go.mod h1:V9rQKRmK7AWuEsOMnHzKj8WyrIir1yUJbZxDuZLFvXI= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.9 h1:w9LnHqTq8MEdlnyhV4Bwfizd65lfNCNgdlNC6mM5paE= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.9/go.mod h1:LGEP6EK4nj+bwWNdrvX/FnDTFowdBNwcSPuZu/ouFys= -github.com/aws/aws-sdk-go-v2/service/dynamodb v1.49.1 h1:0RqS5X7EodJzOenoY4V3LUSp9PirELO2ZOpOZbMldco= -github.com/aws/aws-sdk-go-v2/service/dynamodb v1.49.1/go.mod h1:VRp/OeQolnQD9GfNgdSf3kU5vbg708PF6oPHh2bq3hc= -github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.24.5 h1:pc8+YeYe6bBe8D3QeBz9/S5kUZ9k9yoBMbljGIBMNK4= -github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.24.5/go.mod h1:R09/8/9eLYHJ50PQ8FlIGjZb3XA2t2XhcI5E5332eCI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 h1:oegbebPEMA/1Jny7kvwejowCaHz1FWZAQ94WXFNCyTM= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1/go.mod h1:kemo5Myr9ac0U9JfSjMo9yHLtw+pECEHsFtJ9tqCEI8= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.0 h1:X0FveUndcZ3lKbSpIC6rMYGRiQTcUVRNH6X4yYtIrlU= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.0/go.mod h1:IWjQYlqw4EX9jw2g3qnEPPWvCE6bS8fKzhMed1OK7c8= -github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.4 h1:upi++G3fQCAUBXQe58TbjXmdVPwrqMnRQMThOAIz7KM= -github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.4/go.mod h1:swb+GqWXTZMOyVV9rVePAUu5L80+X5a+Lui1RNOyUFo= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9 h1:5r34CgVOD4WZudeEKZ9/iKpiT6cM1JyEROpXjOcdWv8= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9/go.mod h1:dB12CEbNWPbzO2uC6QSWHteqOg4JfBVJOojbAoAUb5I= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.9 h1:wuZ5uW2uhJR63zwNlqWH2W4aL4ZjeJP3o92/W+odDY4= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.9/go.mod h1:/G58M2fGszCrOzvJUkDdY8O9kycodunH4VdT5oBAqls= github.com/aws/aws-sdk-go-v2/service/s3 v1.88.4 h1:mUI3b885qJgfqKDUSj6RgbRqLdX0wGmg8ruM03zNfQA= github.com/aws/aws-sdk-go-v2/service/s3 v1.88.4/go.mod h1:6v8ukAxc7z4x4oBjGUsLnH7KGLY9Uhcgij19UJNkiMg= -github.com/aws/aws-sdk-go-v2/service/sqs v1.42.8 h1:cWiY+//XL5QOYKJyf4Pvt+oE/5wSIi095+bS+ME2lGw= -github.com/aws/aws-sdk-go-v2/service/sqs v1.42.8/go.mod h1:sLvnKf0p0sMQ33nkJGP2NpYyWHMojpL0O9neiCGc9lc= -github.com/aws/aws-sdk-go-v2/service/ssm v1.55.5 h1:lGHvjwVUclt6xo91f+H0vdVMfCjw2zclL0sVQXgTOp8= -github.com/aws/aws-sdk-go-v2/service/ssm v1.55.5/go.mod h1:zH7gDT/mAjLk10jcoltSXvjruPmvDSpfCTqzA+0B3l4= -github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 h1:ve9dYBB8CfJGTFqcQ3ZLAAb/KXWgYlgu/2R2TZL2Ko0= -github.com/aws/aws-sdk-go-v2/service/sso v1.28.2/go.mod h1:n9bTZFZcBa9hGGqVz3i/a6+NG0zmZgtkB9qVVFDqPA8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2 h1:pd9G9HQaM6UZAZh19pYOkpKSQkyQQ9ftnl/LttQOcGI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2/go.mod h1:eknndR9rU8UpE/OmFpqU78V1EcXPKFTTm5l/buZYgvM= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 h1:iV1Ko4Em/lkJIsoKyGfc0nQySi+v0Udxr6Igq+y9JZc= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.0/go.mod h1:bEPcjW7IbolPfK67G1nilqWyoxYMSPrDiIQ3RdIdKgo= github.com/aws/smithy-go v1.23.0 h1:8n6I3gXzWJB2DxBDnfxgBaSX6oe0d/t10qGz7OKqMCE= github.com/aws/smithy-go v1.23.0/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= -github.com/awslabs/aws-lambda-go-api-proxy v0.16.2 h1:CJyGEyO1CIwOnXTU40urf0mchf6t3voxpvUDikOU9LY= -github.com/awslabs/aws-lambda-go-api-proxy v0.16.2/go.mod h1:vxxjwBHe/KbgFeNlAP/Tvp4SsVRL3WQamcWRxqVh0z0= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= @@ -698,8 +666,6 @@ github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdW github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= -github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= @@ -753,6 +719,7 @@ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9 github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= @@ -1170,10 +1137,6 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= @@ -1494,6 +1457,7 @@ github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a github.com/nkovacs/streamquote v1.0.0 h1:PmVIV08Zlx2lZK5fFZlMZ04eHcDTIFJCv/5/0twVUow= github.com/nkovacs/streamquote v1.0.0/go.mod h1:BN+NaZ2CmdKqUuTUXUEm9j95B2TRbpOWpxbJYzzgUsc= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -1541,8 +1505,6 @@ github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2 github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o= github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M= github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk= @@ -2281,6 +2243,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2464,6 +2427,7 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= diff --git a/internal/mocks/contract_backend.go b/internal/mocks/contract_backend.go deleted file mode 100644 index 04cbde76..00000000 --- a/internal/mocks/contract_backend.go +++ /dev/null @@ -1,209 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/ethereum/go-ethereum/accounts/abi/bind (interfaces: ContractBackend) -// -// Generated by this command: -// -// mockgen -destination=./internal/mocks/contract_backend.go -package=mocks github.com/ethereum/go-ethereum/accounts/abi/bind ContractBackend -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - big "math/big" - reflect "reflect" - - ethereum "github.com/ethereum/go-ethereum" - common "github.com/ethereum/go-ethereum/common" - types "github.com/ethereum/go-ethereum/core/types" - gomock "go.uber.org/mock/gomock" -) - -// MockContractBackend is a mock of ContractBackend interface. -type MockContractBackend struct { - ctrl *gomock.Controller - recorder *MockContractBackendMockRecorder - isgomock struct{} -} - -// MockContractBackendMockRecorder is the mock recorder for MockContractBackend. -type MockContractBackendMockRecorder struct { - mock *MockContractBackend -} - -// NewMockContractBackend creates a new mock instance. -func NewMockContractBackend(ctrl *gomock.Controller) *MockContractBackend { - mock := &MockContractBackend{ctrl: ctrl} - mock.recorder = &MockContractBackendMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockContractBackend) EXPECT() *MockContractBackendMockRecorder { - return m.recorder -} - -// CallContract mocks base method. -func (m *MockContractBackend) CallContract(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CallContract", ctx, call, blockNumber) - ret0, _ := ret[0].([]byte) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CallContract indicates an expected call of CallContract. -func (mr *MockContractBackendMockRecorder) CallContract(ctx, call, blockNumber any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CallContract", reflect.TypeOf((*MockContractBackend)(nil).CallContract), ctx, call, blockNumber) -} - -// CodeAt mocks base method. -func (m *MockContractBackend) CodeAt(ctx context.Context, contract common.Address, blockNumber *big.Int) ([]byte, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CodeAt", ctx, contract, blockNumber) - ret0, _ := ret[0].([]byte) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CodeAt indicates an expected call of CodeAt. -func (mr *MockContractBackendMockRecorder) CodeAt(ctx, contract, blockNumber any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CodeAt", reflect.TypeOf((*MockContractBackend)(nil).CodeAt), ctx, contract, blockNumber) -} - -// EstimateGas mocks base method. -func (m *MockContractBackend) EstimateGas(ctx context.Context, call ethereum.CallMsg) (uint64, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "EstimateGas", ctx, call) - ret0, _ := ret[0].(uint64) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// EstimateGas indicates an expected call of EstimateGas. -func (mr *MockContractBackendMockRecorder) EstimateGas(ctx, call any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EstimateGas", reflect.TypeOf((*MockContractBackend)(nil).EstimateGas), ctx, call) -} - -// FilterLogs mocks base method. -func (m *MockContractBackend) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "FilterLogs", ctx, q) - ret0, _ := ret[0].([]types.Log) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// FilterLogs indicates an expected call of FilterLogs. -func (mr *MockContractBackendMockRecorder) FilterLogs(ctx, q any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FilterLogs", reflect.TypeOf((*MockContractBackend)(nil).FilterLogs), ctx, q) -} - -// HeaderByNumber mocks base method. -func (m *MockContractBackend) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "HeaderByNumber", ctx, number) - ret0, _ := ret[0].(*types.Header) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// HeaderByNumber indicates an expected call of HeaderByNumber. -func (mr *MockContractBackendMockRecorder) HeaderByNumber(ctx, number any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HeaderByNumber", reflect.TypeOf((*MockContractBackend)(nil).HeaderByNumber), ctx, number) -} - -// PendingCodeAt mocks base method. -func (m *MockContractBackend) PendingCodeAt(ctx context.Context, account common.Address) ([]byte, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "PendingCodeAt", ctx, account) - ret0, _ := ret[0].([]byte) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// PendingCodeAt indicates an expected call of PendingCodeAt. -func (mr *MockContractBackendMockRecorder) PendingCodeAt(ctx, account any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PendingCodeAt", reflect.TypeOf((*MockContractBackend)(nil).PendingCodeAt), ctx, account) -} - -// PendingNonceAt mocks base method. -func (m *MockContractBackend) PendingNonceAt(ctx context.Context, account common.Address) (uint64, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "PendingNonceAt", ctx, account) - ret0, _ := ret[0].(uint64) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// PendingNonceAt indicates an expected call of PendingNonceAt. -func (mr *MockContractBackendMockRecorder) PendingNonceAt(ctx, account any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PendingNonceAt", reflect.TypeOf((*MockContractBackend)(nil).PendingNonceAt), ctx, account) -} - -// SendTransaction mocks base method. -func (m *MockContractBackend) SendTransaction(ctx context.Context, tx *types.Transaction) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendTransaction", ctx, tx) - ret0, _ := ret[0].(error) - return ret0 -} - -// SendTransaction indicates an expected call of SendTransaction. -func (mr *MockContractBackendMockRecorder) SendTransaction(ctx, tx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendTransaction", reflect.TypeOf((*MockContractBackend)(nil).SendTransaction), ctx, tx) -} - -// SubscribeFilterLogs mocks base method. -func (m *MockContractBackend) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SubscribeFilterLogs", ctx, q, ch) - ret0, _ := ret[0].(ethereum.Subscription) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// SubscribeFilterLogs indicates an expected call of SubscribeFilterLogs. -func (mr *MockContractBackendMockRecorder) SubscribeFilterLogs(ctx, q, ch any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubscribeFilterLogs", reflect.TypeOf((*MockContractBackend)(nil).SubscribeFilterLogs), ctx, q, ch) -} - -// SuggestGasPrice mocks base method. -func (m *MockContractBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SuggestGasPrice", ctx) - ret0, _ := ret[0].(*big.Int) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// SuggestGasPrice indicates an expected call of SuggestGasPrice. -func (mr *MockContractBackendMockRecorder) SuggestGasPrice(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SuggestGasPrice", reflect.TypeOf((*MockContractBackend)(nil).SuggestGasPrice), ctx) -} - -// SuggestGasTipCap mocks base method. -func (m *MockContractBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SuggestGasTipCap", ctx) - ret0, _ := ret[0].(*big.Int) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// SuggestGasTipCap indicates an expected call of SuggestGasTipCap. -func (mr *MockContractBackendMockRecorder) SuggestGasTipCap(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SuggestGasTipCap", reflect.TypeOf((*MockContractBackend)(nil).SuggestGasTipCap), ctx) -} diff --git a/internal/mocks/ipldstore.go b/internal/mocks/ipldstore.go deleted file mode 100644 index 67ab493e..00000000 --- a/internal/mocks/ipldstore.go +++ /dev/null @@ -1,70 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: ./internal/ipldstore/ipldstore.go -// -// Generated by this command: -// -// mockgen -source=./internal/ipldstore/ipldstore.go -destination=./internal/mocks/ipldstore.go -package=mocks -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - reflect "reflect" - - gomock "go.uber.org/mock/gomock" -) - -// MockKVStore is a mock of KVStore interface. -type MockKVStore[K any, V any] struct { - ctrl *gomock.Controller - recorder *MockKVStoreMockRecorder[K, V] - isgomock struct{} -} - -// MockKVStoreMockRecorder is the mock recorder for MockKVStore. -type MockKVStoreMockRecorder[K any, V any] struct { - mock *MockKVStore[K, V] -} - -// NewMockKVStore creates a new mock instance. -func NewMockKVStore[K any, V any](ctrl *gomock.Controller) *MockKVStore[K, V] { - mock := &MockKVStore[K, V]{ctrl: ctrl} - mock.recorder = &MockKVStoreMockRecorder[K, V]{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockKVStore[K, V]) EXPECT() *MockKVStoreMockRecorder[K, V] { - return m.recorder -} - -// Get mocks base method. -func (m *MockKVStore[K, V]) Get(ctx context.Context, key K) (V, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Get", ctx, key) - ret0, _ := ret[0].(V) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Get indicates an expected call of Get. -func (mr *MockKVStoreMockRecorder[K, V]) Get(ctx, key any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockKVStore[K, V])(nil).Get), ctx, key) -} - -// Put mocks base method. -func (m *MockKVStore[K, V]) Put(ctx context.Context, key K, value V) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Put", ctx, key, value) - ret0, _ := ret[0].(error) - return ret0 -} - -// Put indicates an expected call of Put. -func (mr *MockKVStoreMockRecorder[K, V]) Put(ctx, key, value any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockKVStore[K, V])(nil).Put), ctx, key, value) -} diff --git a/internal/mocks/message_watcher_eth_client.go b/internal/mocks/message_watcher_eth_client.go deleted file mode 100644 index 78d502d7..00000000 --- a/internal/mocks/message_watcher_eth_client.go +++ /dev/null @@ -1,74 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/storacha/piri/pkg/pdp/tasks (interfaces: MessageWatcherEthClient) -// -// Generated by this command: -// -// mockgen -destination=./internal/mocks/message_watcher_eth_client.go -package=mocks github.com/storacha/piri/pkg/pdp/tasks MessageWatcherEthClient -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - reflect "reflect" - - common "github.com/ethereum/go-ethereum/common" - types "github.com/ethereum/go-ethereum/core/types" - gomock "go.uber.org/mock/gomock" -) - -// MockMessageWatcherEthClient is a mock of MessageWatcherEthClient interface. -type MockMessageWatcherEthClient struct { - ctrl *gomock.Controller - recorder *MockMessageWatcherEthClientMockRecorder - isgomock struct{} -} - -// MockMessageWatcherEthClientMockRecorder is the mock recorder for MockMessageWatcherEthClient. -type MockMessageWatcherEthClientMockRecorder struct { - mock *MockMessageWatcherEthClient -} - -// NewMockMessageWatcherEthClient creates a new mock instance. -func NewMockMessageWatcherEthClient(ctrl *gomock.Controller) *MockMessageWatcherEthClient { - mock := &MockMessageWatcherEthClient{ctrl: ctrl} - mock.recorder = &MockMessageWatcherEthClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockMessageWatcherEthClient) EXPECT() *MockMessageWatcherEthClientMockRecorder { - return m.recorder -} - -// TransactionByHash mocks base method. -func (m *MockMessageWatcherEthClient) TransactionByHash(ctx context.Context, hash common.Hash) (*types.Transaction, bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "TransactionByHash", ctx, hash) - ret0, _ := ret[0].(*types.Transaction) - ret1, _ := ret[1].(bool) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// TransactionByHash indicates an expected call of TransactionByHash. -func (mr *MockMessageWatcherEthClientMockRecorder) TransactionByHash(ctx, hash any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TransactionByHash", reflect.TypeOf((*MockMessageWatcherEthClient)(nil).TransactionByHash), ctx, hash) -} - -// TransactionReceipt mocks base method. -func (m *MockMessageWatcherEthClient) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "TransactionReceipt", ctx, txHash) - ret0, _ := ret[0].(*types.Receipt) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// TransactionReceipt indicates an expected call of TransactionReceipt. -func (mr *MockMessageWatcherEthClientMockRecorder) TransactionReceipt(ctx, txHash any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TransactionReceipt", reflect.TypeOf((*MockMessageWatcherEthClient)(nil).TransactionReceipt), ctx, txHash) -} diff --git a/internal/mocks/pdp_api.go b/internal/mocks/pdp_api.go deleted file mode 100644 index b0f857ca..00000000 --- a/internal/mocks/pdp_api.go +++ /dev/null @@ -1,372 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: ./pkg/pdp/types/api.go -// -// Generated by this command: -// -// mockgen -source=./pkg/pdp/types/api.go -destination=./internal/mocks/pdp_api.go -package=mocks -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - reflect "reflect" - - common "github.com/ethereum/go-ethereum/common" - cid "github.com/ipfs/go-cid" - types "github.com/storacha/piri/pkg/pdp/types" - gomock "go.uber.org/mock/gomock" -) - -// MockAPI is a mock of API interface. -type MockAPI struct { - ctrl *gomock.Controller - recorder *MockAPIMockRecorder - isgomock struct{} -} - -// MockAPIMockRecorder is the mock recorder for MockAPI. -type MockAPIMockRecorder struct { - mock *MockAPI -} - -// NewMockAPI creates a new mock instance. -func NewMockAPI(ctrl *gomock.Controller) *MockAPI { - mock := &MockAPI{ctrl: ctrl} - mock.recorder = &MockAPIMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockAPI) EXPECT() *MockAPIMockRecorder { - return m.recorder -} - -// AddRoots mocks base method. -func (m *MockAPI) AddRoots(ctx context.Context, proofSetID uint64, roots []types.RootAdd) (common.Hash, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddRoots", ctx, proofSetID, roots) - ret0, _ := ret[0].(common.Hash) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// AddRoots indicates an expected call of AddRoots. -func (mr *MockAPIMockRecorder) AddRoots(ctx, proofSetID, roots any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRoots", reflect.TypeOf((*MockAPI)(nil).AddRoots), ctx, proofSetID, roots) -} - -// AllocatePiece mocks base method. -func (m *MockAPI) AllocatePiece(ctx context.Context, allocation types.PieceAllocation) (*types.AllocatedPiece, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AllocatePiece", ctx, allocation) - ret0, _ := ret[0].(*types.AllocatedPiece) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// AllocatePiece indicates an expected call of AllocatePiece. -func (mr *MockAPIMockRecorder) AllocatePiece(ctx, allocation any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocatePiece", reflect.TypeOf((*MockAPI)(nil).AllocatePiece), ctx, allocation) -} - -// CreateProofSet mocks base method. -func (m *MockAPI) CreateProofSet(ctx context.Context, recordKeeper common.Address) (common.Hash, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateProofSet", ctx, recordKeeper) - ret0, _ := ret[0].(common.Hash) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateProofSet indicates an expected call of CreateProofSet. -func (mr *MockAPIMockRecorder) CreateProofSet(ctx, recordKeeper any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateProofSet", reflect.TypeOf((*MockAPI)(nil).CreateProofSet), ctx, recordKeeper) -} - -// FindPiece mocks base method. -func (m *MockAPI) FindPiece(ctx context.Context, piece types.Piece) (cid.Cid, bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "FindPiece", ctx, piece) - ret0, _ := ret[0].(cid.Cid) - ret1, _ := ret[1].(bool) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// FindPiece indicates an expected call of FindPiece. -func (mr *MockAPIMockRecorder) FindPiece(ctx, piece any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindPiece", reflect.TypeOf((*MockAPI)(nil).FindPiece), ctx, piece) -} - -// GetProofSet mocks base method. -func (m *MockAPI) GetProofSet(ctx context.Context, proofSetID uint64) (*types.ProofSet, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetProofSet", ctx, proofSetID) - ret0, _ := ret[0].(*types.ProofSet) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetProofSet indicates an expected call of GetProofSet. -func (mr *MockAPIMockRecorder) GetProofSet(ctx, proofSetID any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProofSet", reflect.TypeOf((*MockAPI)(nil).GetProofSet), ctx, proofSetID) -} - -// GetProofSetStatus mocks base method. -func (m *MockAPI) GetProofSetStatus(ctx context.Context, txHash common.Hash) (*types.ProofSetStatus, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetProofSetStatus", ctx, txHash) - ret0, _ := ret[0].(*types.ProofSetStatus) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetProofSetStatus indicates an expected call of GetProofSetStatus. -func (mr *MockAPIMockRecorder) GetProofSetStatus(ctx, txHash any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProofSetStatus", reflect.TypeOf((*MockAPI)(nil).GetProofSetStatus), ctx, txHash) -} - -// ReadPiece mocks base method. -func (m *MockAPI) ReadPiece(ctx context.Context, piece cid.Cid, options ...types.ReadPieceOption) (*types.PieceReader, error) { - m.ctrl.T.Helper() - varargs := []any{ctx, piece} - for _, a := range options { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "ReadPiece", varargs...) - ret0, _ := ret[0].(*types.PieceReader) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ReadPiece indicates an expected call of ReadPiece. -func (mr *MockAPIMockRecorder) ReadPiece(ctx, piece any, options ...any) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]any{ctx, piece}, options...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadPiece", reflect.TypeOf((*MockAPI)(nil).ReadPiece), varargs...) -} - -// RemoveRoot mocks base method. -func (m *MockAPI) RemoveRoot(ctx context.Context, proofSetID, rootID uint64) (common.Hash, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RemoveRoot", ctx, proofSetID, rootID) - ret0, _ := ret[0].(common.Hash) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// RemoveRoot indicates an expected call of RemoveRoot. -func (mr *MockAPIMockRecorder) RemoveRoot(ctx, proofSetID, rootID any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveRoot", reflect.TypeOf((*MockAPI)(nil).RemoveRoot), ctx, proofSetID, rootID) -} - -// UploadPiece mocks base method. -func (m *MockAPI) UploadPiece(ctx context.Context, upload types.PieceUpload) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UploadPiece", ctx, upload) - ret0, _ := ret[0].(error) - return ret0 -} - -// UploadPiece indicates an expected call of UploadPiece. -func (mr *MockAPIMockRecorder) UploadPiece(ctx, upload any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UploadPiece", reflect.TypeOf((*MockAPI)(nil).UploadPiece), ctx, upload) -} - -// MockProofSetAPI is a mock of ProofSetAPI interface. -type MockProofSetAPI struct { - ctrl *gomock.Controller - recorder *MockProofSetAPIMockRecorder - isgomock struct{} -} - -// MockProofSetAPIMockRecorder is the mock recorder for MockProofSetAPI. -type MockProofSetAPIMockRecorder struct { - mock *MockProofSetAPI -} - -// NewMockProofSetAPI creates a new mock instance. -func NewMockProofSetAPI(ctrl *gomock.Controller) *MockProofSetAPI { - mock := &MockProofSetAPI{ctrl: ctrl} - mock.recorder = &MockProofSetAPIMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockProofSetAPI) EXPECT() *MockProofSetAPIMockRecorder { - return m.recorder -} - -// AddRoots mocks base method. -func (m *MockProofSetAPI) AddRoots(ctx context.Context, proofSetID uint64, roots []types.RootAdd) (common.Hash, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddRoots", ctx, proofSetID, roots) - ret0, _ := ret[0].(common.Hash) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// AddRoots indicates an expected call of AddRoots. -func (mr *MockProofSetAPIMockRecorder) AddRoots(ctx, proofSetID, roots any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRoots", reflect.TypeOf((*MockProofSetAPI)(nil).AddRoots), ctx, proofSetID, roots) -} - -// CreateProofSet mocks base method. -func (m *MockProofSetAPI) CreateProofSet(ctx context.Context, recordKeeper common.Address) (common.Hash, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateProofSet", ctx, recordKeeper) - ret0, _ := ret[0].(common.Hash) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateProofSet indicates an expected call of CreateProofSet. -func (mr *MockProofSetAPIMockRecorder) CreateProofSet(ctx, recordKeeper any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateProofSet", reflect.TypeOf((*MockProofSetAPI)(nil).CreateProofSet), ctx, recordKeeper) -} - -// GetProofSet mocks base method. -func (m *MockProofSetAPI) GetProofSet(ctx context.Context, proofSetID uint64) (*types.ProofSet, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetProofSet", ctx, proofSetID) - ret0, _ := ret[0].(*types.ProofSet) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetProofSet indicates an expected call of GetProofSet. -func (mr *MockProofSetAPIMockRecorder) GetProofSet(ctx, proofSetID any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProofSet", reflect.TypeOf((*MockProofSetAPI)(nil).GetProofSet), ctx, proofSetID) -} - -// GetProofSetStatus mocks base method. -func (m *MockProofSetAPI) GetProofSetStatus(ctx context.Context, txHash common.Hash) (*types.ProofSetStatus, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetProofSetStatus", ctx, txHash) - ret0, _ := ret[0].(*types.ProofSetStatus) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetProofSetStatus indicates an expected call of GetProofSetStatus. -func (mr *MockProofSetAPIMockRecorder) GetProofSetStatus(ctx, txHash any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProofSetStatus", reflect.TypeOf((*MockProofSetAPI)(nil).GetProofSetStatus), ctx, txHash) -} - -// RemoveRoot mocks base method. -func (m *MockProofSetAPI) RemoveRoot(ctx context.Context, proofSetID, rootID uint64) (common.Hash, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RemoveRoot", ctx, proofSetID, rootID) - ret0, _ := ret[0].(common.Hash) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// RemoveRoot indicates an expected call of RemoveRoot. -func (mr *MockProofSetAPIMockRecorder) RemoveRoot(ctx, proofSetID, rootID any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveRoot", reflect.TypeOf((*MockProofSetAPI)(nil).RemoveRoot), ctx, proofSetID, rootID) -} - -// MockPieceAPI is a mock of PieceAPI interface. -type MockPieceAPI struct { - ctrl *gomock.Controller - recorder *MockPieceAPIMockRecorder - isgomock struct{} -} - -// MockPieceAPIMockRecorder is the mock recorder for MockPieceAPI. -type MockPieceAPIMockRecorder struct { - mock *MockPieceAPI -} - -// NewMockPieceAPI creates a new mock instance. -func NewMockPieceAPI(ctrl *gomock.Controller) *MockPieceAPI { - mock := &MockPieceAPI{ctrl: ctrl} - mock.recorder = &MockPieceAPIMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockPieceAPI) EXPECT() *MockPieceAPIMockRecorder { - return m.recorder -} - -// AllocatePiece mocks base method. -func (m *MockPieceAPI) AllocatePiece(ctx context.Context, allocation types.PieceAllocation) (*types.AllocatedPiece, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AllocatePiece", ctx, allocation) - ret0, _ := ret[0].(*types.AllocatedPiece) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// AllocatePiece indicates an expected call of AllocatePiece. -func (mr *MockPieceAPIMockRecorder) AllocatePiece(ctx, allocation any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocatePiece", reflect.TypeOf((*MockPieceAPI)(nil).AllocatePiece), ctx, allocation) -} - -// FindPiece mocks base method. -func (m *MockPieceAPI) FindPiece(ctx context.Context, piece types.Piece) (cid.Cid, bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "FindPiece", ctx, piece) - ret0, _ := ret[0].(cid.Cid) - ret1, _ := ret[1].(bool) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// FindPiece indicates an expected call of FindPiece. -func (mr *MockPieceAPIMockRecorder) FindPiece(ctx, piece any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindPiece", reflect.TypeOf((*MockPieceAPI)(nil).FindPiece), ctx, piece) -} - -// ReadPiece mocks base method. -func (m *MockPieceAPI) ReadPiece(ctx context.Context, piece cid.Cid, options ...types.ReadPieceOption) (*types.PieceReader, error) { - m.ctrl.T.Helper() - varargs := []any{ctx, piece} - for _, a := range options { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "ReadPiece", varargs...) - ret0, _ := ret[0].(*types.PieceReader) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ReadPiece indicates an expected call of ReadPiece. -func (mr *MockPieceAPIMockRecorder) ReadPiece(ctx, piece any, options ...any) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]any{ctx, piece}, options...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadPiece", reflect.TypeOf((*MockPieceAPI)(nil).ReadPiece), varargs...) -} - -// UploadPiece mocks base method. -func (m *MockPieceAPI) UploadPiece(ctx context.Context, upload types.PieceUpload) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UploadPiece", ctx, upload) - ret0, _ := ret[0].(error) - return ret0 -} - -// UploadPiece indicates an expected call of UploadPiece. -func (mr *MockPieceAPIMockRecorder) UploadPiece(ctx, upload any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UploadPiece", reflect.TypeOf((*MockPieceAPI)(nil).UploadPiece), ctx, upload) -} diff --git a/internal/mocks/sender_eth_client.go b/internal/mocks/sender_eth_client.go deleted file mode 100644 index 79cb6a79..00000000 --- a/internal/mocks/sender_eth_client.go +++ /dev/null @@ -1,134 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/storacha/piri/pkg/pdp/tasks (interfaces: SenderETHClient) -// -// Generated by this command: -// -// mockgen -destination=./internal/mocks/sender_eth_client.go -package=mocks github.com/storacha/piri/pkg/pdp/tasks SenderETHClient -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - big "math/big" - reflect "reflect" - - ethereum "github.com/ethereum/go-ethereum" - common "github.com/ethereum/go-ethereum/common" - types "github.com/ethereum/go-ethereum/core/types" - gomock "go.uber.org/mock/gomock" -) - -// MockSenderETHClient is a mock of SenderETHClient interface. -type MockSenderETHClient struct { - ctrl *gomock.Controller - recorder *MockSenderETHClientMockRecorder - isgomock struct{} -} - -// MockSenderETHClientMockRecorder is the mock recorder for MockSenderETHClient. -type MockSenderETHClientMockRecorder struct { - mock *MockSenderETHClient -} - -// NewMockSenderETHClient creates a new mock instance. -func NewMockSenderETHClient(ctrl *gomock.Controller) *MockSenderETHClient { - mock := &MockSenderETHClient{ctrl: ctrl} - mock.recorder = &MockSenderETHClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockSenderETHClient) EXPECT() *MockSenderETHClientMockRecorder { - return m.recorder -} - -// EstimateGas mocks base method. -func (m *MockSenderETHClient) EstimateGas(ctx context.Context, msg ethereum.CallMsg) (uint64, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "EstimateGas", ctx, msg) - ret0, _ := ret[0].(uint64) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// EstimateGas indicates an expected call of EstimateGas. -func (mr *MockSenderETHClientMockRecorder) EstimateGas(ctx, msg any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EstimateGas", reflect.TypeOf((*MockSenderETHClient)(nil).EstimateGas), ctx, msg) -} - -// HeaderByNumber mocks base method. -func (m *MockSenderETHClient) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "HeaderByNumber", ctx, number) - ret0, _ := ret[0].(*types.Header) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// HeaderByNumber indicates an expected call of HeaderByNumber. -func (mr *MockSenderETHClientMockRecorder) HeaderByNumber(ctx, number any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HeaderByNumber", reflect.TypeOf((*MockSenderETHClient)(nil).HeaderByNumber), ctx, number) -} - -// NetworkID mocks base method. -func (m *MockSenderETHClient) NetworkID(ctx context.Context) (*big.Int, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NetworkID", ctx) - ret0, _ := ret[0].(*big.Int) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// NetworkID indicates an expected call of NetworkID. -func (mr *MockSenderETHClientMockRecorder) NetworkID(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetworkID", reflect.TypeOf((*MockSenderETHClient)(nil).NetworkID), ctx) -} - -// PendingNonceAt mocks base method. -func (m *MockSenderETHClient) PendingNonceAt(ctx context.Context, account common.Address) (uint64, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "PendingNonceAt", ctx, account) - ret0, _ := ret[0].(uint64) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// PendingNonceAt indicates an expected call of PendingNonceAt. -func (mr *MockSenderETHClientMockRecorder) PendingNonceAt(ctx, account any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PendingNonceAt", reflect.TypeOf((*MockSenderETHClient)(nil).PendingNonceAt), ctx, account) -} - -// SendTransaction mocks base method. -func (m *MockSenderETHClient) SendTransaction(ctx context.Context, transaction *types.Transaction) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendTransaction", ctx, transaction) - ret0, _ := ret[0].(error) - return ret0 -} - -// SendTransaction indicates an expected call of SendTransaction. -func (mr *MockSenderETHClientMockRecorder) SendTransaction(ctx, transaction any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendTransaction", reflect.TypeOf((*MockSenderETHClient)(nil).SendTransaction), ctx, transaction) -} - -// SuggestGasTipCap mocks base method. -func (m *MockSenderETHClient) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SuggestGasTipCap", ctx) - ret0, _ := ret[0].(*big.Int) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// SuggestGasTipCap indicates an expected call of SuggestGasTipCap. -func (mr *MockSenderETHClientMockRecorder) SuggestGasTipCap(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SuggestGasTipCap", reflect.TypeOf((*MockSenderETHClient)(nil).SuggestGasTipCap), ctx) -} diff --git a/internal/telemetry/errors.go b/internal/telemetry/errors.go deleted file mode 100644 index c64abd3a..00000000 --- a/internal/telemetry/errors.go +++ /dev/null @@ -1,81 +0,0 @@ -package telemetry - -import ( - "context" - "log" - "net/http" - - "github.com/getsentry/sentry-go" - sentryhttp "github.com/getsentry/sentry-go/http" - "github.com/labstack/echo/v4" - "github.com/storacha/piri/pkg/build" -) - -// HTTPError is an error that also has an associated HTTP status code -type HTTPError struct { - err error - statusCode int -} - -// Error implements the error interface -func (he HTTPError) Error() string { - return he.err.Error() -} - -// StatusCode returns the HTTP status code associated with the error -func (he HTTPError) StatusCode() int { - return he.statusCode -} - -// NewHTTPError creates a new HTTPError -func NewHTTPError(err error, statusCode int) HTTPError { - return HTTPError{err: err, statusCode: statusCode} -} - -// ErrorReturningHTTPHandler is a HTTP handler function that returns an error -type ErrorReturningHTTPHandler func(http.ResponseWriter, *http.Request) error - -// SetupErrorReporting configures the Sentry SDK for error reporting -func SetupErrorReporting(sentryDSN, environment string) { - err := sentry.Init(sentry.ClientOptions{ - Dsn: sentryDSN, - Environment: environment, - Release: build.Version, - Transport: sentry.NewHTTPSyncTransport(), - EnableTracing: false, - }) - - if err != nil { - log.Fatalf("sentry.Init: %s", err) - } -} - -// NewErrorReportingHandler wraps an ErrorReturningHTTPHandler with error reporting -func NewErrorReportingHandler(errorReturningHandler ErrorReturningHTTPHandler) http.Handler { - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := errorReturningHandler(w, r); err != nil { - ReportError(r.Context(), err) - - // if the error is an HTTPError or *echo.HTTPError, send an appropriate - // response as well as reporting it - if httperr, ok := err.(*echo.HTTPError); ok { - http.Error(w, http.StatusText(httperr.Code), httperr.Code) - } else if e, ok := err.(HTTPError); ok { - http.Error(w, e.Error(), e.StatusCode()) - } - } - }) - - sentryHandler := sentryhttp.New(sentryhttp.Options{}) - return sentryHandler.Handle(handler) -} - -// ReportError reports an error to Sentry -func ReportError(ctx context.Context, err error) { - hub := sentry.GetHubFromContext(ctx) - if hub != nil { - hub.CaptureException(err) - } else { - sentry.CaptureException(err) - } -} diff --git a/pkg/aws/dynamoacceptancestore.go b/pkg/aws/dynamoacceptancestore.go deleted file mode 100644 index a437dcb0..00000000 --- a/pkg/aws/dynamoacceptancestore.go +++ /dev/null @@ -1,154 +0,0 @@ -package aws - -import ( - "context" - "fmt" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" - "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" - "github.com/aws/aws-sdk-go-v2/service/dynamodb" - "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/ipld/go-ipld-prime/codec/dagcbor" - multihash "github.com/multiformats/go-multihash" - "github.com/storacha/go-libstoracha/digestutil" - "github.com/storacha/go-ucanto/did" - "github.com/storacha/piri/pkg/store" - "github.com/storacha/piri/pkg/store/acceptancestore" - "github.com/storacha/piri/pkg/store/acceptancestore/acceptance" -) - -// DynamoAcceptanceStore implements the AcceptanceStore interface on dynamodb -type DynamoAcceptanceStore struct { - tableName string - dynamoDbClient *dynamodb.Client -} - -// NewDynamoAcceptanceStore returns an AcceptanceStore connected to a AWS DynamoDB table -func NewDynamoAcceptanceStore(cfg aws.Config, tableName string, opts ...func(*dynamodb.Options)) *DynamoAcceptanceStore { - return &DynamoAcceptanceStore{ - tableName: tableName, - dynamoDbClient: dynamodb.NewFromConfig(cfg, opts...), - } -} - -func (d *DynamoAcceptanceStore) Get(ctx context.Context, mh multihash.Multihash, space did.DID) (acceptance.Acceptance, error) { - res, err := d.dynamoDbClient.GetItem(ctx, &dynamodb.GetItemInput{ - TableName: aws.String(d.tableName), - Key: map[string]types.AttributeValue{ - "hash": &types.AttributeValueMemberS{Value: digestutil.Format(mh)}, - "space": &types.AttributeValueMemberS{Value: space.String()}, - }, - }) - if err != nil { - return acceptance.Acceptance{}, fmt.Errorf("getting item: %w", err) - } - - if res.Item == nil { - return acceptance.Acceptance{}, store.ErrNotFound - } - var item acceptanceItem - err = attributevalue.UnmarshalMap(res.Item, &item) - if err != nil { - return acceptance.Acceptance{}, fmt.Errorf("unmarshalling acceptance item: %w", err) - } - acc, err := acceptance.Decode(item.Acceptance, dagcbor.Decode) - if err != nil { - return acceptance.Acceptance{}, fmt.Errorf("decoding acceptance: %w", err) - } - return acc, nil -} - -// GetAny retrieves any acceptance for a blob (digest), regardless of space. -func (d *DynamoAcceptanceStore) GetAny(ctx context.Context, mh multihash.Multihash) (acceptance.Acceptance, error) { - keyEx := expression.Key("hash").Equal(expression.Value(digestutil.Format(mh))) - expr, err := expression.NewBuilder().WithKeyCondition(keyEx).Build() - if err != nil { - return acceptance.Acceptance{}, fmt.Errorf("building query: %w", err) - } - - // Query for just one item - response, err := d.dynamoDbClient.Query(ctx, &dynamodb.QueryInput{ - TableName: aws.String(d.tableName), - ExpressionAttributeNames: expr.Names(), - ExpressionAttributeValues: expr.Values(), - KeyConditionExpression: expr.KeyCondition(), - ConsistentRead: aws.Bool(true), - Limit: aws.Int32(1), - }) - if err != nil { - return acceptance.Acceptance{}, fmt.Errorf("querying acceptances: %w", err) - } - - if len(response.Items) == 0 { - return acceptance.Acceptance{}, store.ErrNotFound - } - - var item acceptanceItem - err = attributevalue.UnmarshalMap(response.Items[0], &item) - if err != nil { - return acceptance.Acceptance{}, fmt.Errorf("parsing query response: %w", err) - } - - acc, err := acceptance.Decode(item.Acceptance, dagcbor.Decode) - if err != nil { - return acceptance.Acceptance{}, fmt.Errorf("decoding data: %w", err) - } - return acc, nil -} - -// Exists checks if any acceptance exists for a blob (digest). -func (d *DynamoAcceptanceStore) Exists(ctx context.Context, mh multihash.Multihash) (bool, error) { - keyEx := expression.Key("hash").Equal(expression.Value(digestutil.Format(mh))) - proj := expression.NamesList(expression.Name("hash")) - expr, err := expression.NewBuilder().WithKeyCondition(keyEx).WithProjection(proj).Build() - if err != nil { - return false, fmt.Errorf("building query: %w", err) - } - - response, err := d.dynamoDbClient.Query(ctx, &dynamodb.QueryInput{ - TableName: aws.String(d.tableName), - ExpressionAttributeNames: expr.Names(), - ExpressionAttributeValues: expr.Values(), - KeyConditionExpression: expr.KeyCondition(), - ProjectionExpression: expr.Projection(), - ConsistentRead: aws.Bool(true), - Limit: aws.Int32(1), - }) - if err != nil { - return false, fmt.Errorf("querying acceptances: %w", err) - } - - return len(response.Items) > 0, nil -} - -// Put implements acceptancestore.AcceptanceStore. -func (d *DynamoAcceptanceStore) Put(ctx context.Context, acc acceptance.Acceptance) error { - data, err := acceptance.Encode(acc, dagcbor.Encode) - if err != nil { - return fmt.Errorf("encoding data: %w", err) - } - item, err := attributevalue.MarshalMap(acceptanceItem{ - Hash: digestutil.Format(acc.Blob.Digest), - Space: acc.Space.String(), - Acceptance: data, - }) - if err != nil { - return fmt.Errorf("serializing item: %w", err) - } - _, err = d.dynamoDbClient.PutItem(ctx, &dynamodb.PutItemInput{ - TableName: aws.String(d.tableName), Item: item, - }) - if err != nil { - return fmt.Errorf("storing item: %w", err) - } - return nil -} - -type acceptanceItem struct { - Hash string `dynamodbav:"hash"` - Space string `dynamodbav:"space"` - Acceptance []byte `dynamodbav:"acceptance"` -} - -var _ acceptancestore.AcceptanceStore = (*DynamoAcceptanceStore)(nil) diff --git a/pkg/aws/dynamoallocationstore.go b/pkg/aws/dynamoallocationstore.go deleted file mode 100644 index 886819ca..00000000 --- a/pkg/aws/dynamoallocationstore.go +++ /dev/null @@ -1,241 +0,0 @@ -package aws - -import ( - "context" - "fmt" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" - "github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression" - "github.com/aws/aws-sdk-go-v2/service/dynamodb" - "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/ipld/go-ipld-prime/codec/dagcbor" - "github.com/multiformats/go-multihash" - "github.com/storacha/go-libstoracha/digestutil" - "github.com/storacha/go-ucanto/did" - - "github.com/storacha/piri/pkg/store" - "github.com/storacha/piri/pkg/store/allocationstore" - "github.com/storacha/piri/pkg/store/allocationstore/allocation" -) - -// DynamoAllocationStore implements the AllocationStore interface on dynamodb -type DynamoAllocationStore struct { - tableName string - dynamoDbClient *dynamodb.Client -} - -// NewDynamoAllocationStore returns an AllocationStore connected to a AWS DynamoDB table -func NewDynamoAllocationStore(cfg aws.Config, tableName string, opts ...func(*dynamodb.Options)) *DynamoAllocationStore { - return &DynamoAllocationStore{ - tableName: tableName, - dynamoDbClient: dynamodb.NewFromConfig(cfg, opts...), - } -} - -func (d *DynamoAllocationStore) Get(ctx context.Context, mh multihash.Multihash, space did.DID) (allocation.Allocation, error) { - res, err := d.dynamoDbClient.GetItem(ctx, &dynamodb.GetItemInput{ - TableName: aws.String(d.tableName), - Key: map[string]types.AttributeValue{ - "hash": &types.AttributeValueMemberS{Value: digestutil.Format(mh)}, - "cause": &types.AttributeValueMemberS{Value: space.String()}, - }, - }) - if err != nil { - return allocation.Allocation{}, fmt.Errorf("getting item: %w", err) - } - - // HACK: (ash) Temporary hack to allow allocation to be found if it was - // stored with the old style key ("/") not the new key - // ("/"). This works because listing works on digest - // prefix i.e. "/*". - if res.Item == nil { - allocs, listErr := d.list(ctx, mh) - if listErr != nil { - return allocation.Allocation{}, fmt.Errorf("listing items: %w", listErr) - } - for _, a := range allocs { - if a.Space == space { - return a, nil - } - } - } - - if res.Item == nil { - return allocation.Allocation{}, store.ErrNotFound - } - var item allocationItem - err = attributevalue.UnmarshalMap(res.Item, &item) - if err != nil { - return allocation.Allocation{}, fmt.Errorf("unmarshalling allocation item: %w", err) - } - alloc, err := allocation.Decode(item.Allocation, dagcbor.Decode) - if err != nil { - return allocation.Allocation{}, fmt.Errorf("decoding allocation: %w", err) - } - return alloc, nil -} - -// GetAny retrieves any allocation for a blob (digest), regardless of space. -func (d *DynamoAllocationStore) GetAny(ctx context.Context, mh multihash.Multihash) (allocation.Allocation, error) { - keyEx := expression.Key("hash").Equal(expression.Value(digestutil.Format(mh))) - expr, err := expression.NewBuilder().WithKeyCondition(keyEx).Build() - if err != nil { - return allocation.Allocation{}, fmt.Errorf("building query: %w", err) - } - - res, err := d.dynamoDbClient.Query(ctx, &dynamodb.QueryInput{ - TableName: aws.String(d.tableName), - ExpressionAttributeNames: expr.Names(), - ExpressionAttributeValues: expr.Values(), - KeyConditionExpression: expr.KeyCondition(), - ConsistentRead: aws.Bool(true), - Limit: aws.Int32(1), - }) - if err != nil { - return allocation.Allocation{}, fmt.Errorf("querying allocation: %w", err) - } - if len(res.Items) == 0 { - return allocation.Allocation{}, store.ErrNotFound - } - - var item allocationItem - err = attributevalue.UnmarshalMap(res.Items[0], &item) - if err != nil { - return allocation.Allocation{}, fmt.Errorf("unmarshalling allocation item: %w", err) - } - alloc, err := allocation.Decode(item.Allocation, dagcbor.Decode) - if err != nil { - return allocation.Allocation{}, fmt.Errorf("decoding allocation: %w", err) - } - return alloc, nil -} - -// GetAnyNonExpired retrieves any allocation for a blob that has not expired. -func (d *DynamoAllocationStore) GetAnyNonExpired(ctx context.Context, mh multihash.Multihash, now uint64) (allocation.Allocation, error) { - keyEx := expression.Key("hash").Equal(expression.Value(digestutil.Format(mh))) - expr, err := expression.NewBuilder().WithKeyCondition(keyEx).Build() - if err != nil { - return allocation.Allocation{}, fmt.Errorf("building query: %w", err) - } - - res, err := d.dynamoDbClient.Query(ctx, &dynamodb.QueryInput{ - TableName: aws.String(d.tableName), - ExpressionAttributeNames: expr.Names(), - ExpressionAttributeValues: expr.Values(), - KeyConditionExpression: expr.KeyCondition(), - ConsistentRead: aws.Bool(true), - }) - if err != nil { - return allocation.Allocation{}, fmt.Errorf("querying allocation: %w", err) - } - - for _, rawItem := range res.Items { - var item allocationItem - err = attributevalue.UnmarshalMap(rawItem, &item) - if err != nil { - return allocation.Allocation{}, fmt.Errorf("unmarshalling allocation item: %w", err) - } - alloc, err := allocation.Decode(item.Allocation, dagcbor.Decode) - if err != nil { - return allocation.Allocation{}, fmt.Errorf("decoding allocation: %w", err) - } - if alloc.Expires > now { - return alloc, nil - } - } - - return allocation.Allocation{}, store.ErrNotFound -} - -// Exists checks if any allocation exists for a blob (digest). -func (d *DynamoAllocationStore) Exists(ctx context.Context, mh multihash.Multihash) (bool, error) { - keyEx := expression.Key("hash").Equal(expression.Value(digestutil.Format(mh))) - proj := expression.NamesList(expression.Name("hash")) - expr, err := expression.NewBuilder().WithKeyCondition(keyEx).WithProjection(proj).Build() - if err != nil { - return false, fmt.Errorf("building query: %w", err) - } - - res, err := d.dynamoDbClient.Query(ctx, &dynamodb.QueryInput{ - TableName: aws.String(d.tableName), - ExpressionAttributeNames: expr.Names(), - ExpressionAttributeValues: expr.Values(), - KeyConditionExpression: expr.KeyCondition(), - ProjectionExpression: expr.Projection(), - ConsistentRead: aws.Bool(true), - Limit: aws.Int32(1), - }) - if err != nil { - return false, fmt.Errorf("querying allocation: %w", err) - } - return len(res.Items) > 0, nil -} - -// Put implements allocationstore.AllocationStore. -func (d *DynamoAllocationStore) Put(ctx context.Context, alloc allocation.Allocation) error { - data, err := allocation.Encode(alloc, dagcbor.Encode) - if err != nil { - return fmt.Errorf("encoding data: %w", err) - } - item, err := attributevalue.MarshalMap(allocationItem{ - Hash: digestutil.Format(alloc.Blob.Digest), - Cause: alloc.Space.String(), - Allocation: data, - }) - if err != nil { - return fmt.Errorf("serializing item: %w", err) - } - _, err = d.dynamoDbClient.PutItem(ctx, &dynamodb.PutItemInput{ - TableName: aws.String(d.tableName), Item: item, - }) - if err != nil { - return fmt.Errorf("storing item: %w", err) - } - return nil -} - -func (d *DynamoAllocationStore) list(ctx context.Context, mh multihash.Multihash) ([]allocation.Allocation, error) { - keyEx := expression.Key("hash").Equal(expression.Value(digestutil.Format(mh))) - expr, err := expression.NewBuilder().WithKeyCondition(keyEx).Build() - if err != nil { - return nil, fmt.Errorf("building query: %w", err) - } - - var allocations []allocation.Allocation - queryPaginator := dynamodb.NewQueryPaginator(d.dynamoDbClient, &dynamodb.QueryInput{ - TableName: aws.String(d.tableName), - ExpressionAttributeNames: expr.Names(), - ExpressionAttributeValues: expr.Values(), - KeyConditionExpression: expr.KeyCondition(), - ConsistentRead: aws.Bool(true), - }) - for queryPaginator.HasMorePages() { - response, err := queryPaginator.NextPage(ctx) - if err != nil { - return nil, fmt.Errorf("querying allocations: %w", err) - } - var allocationPage []allocationItem - err = attributevalue.UnmarshalListOfMaps(response.Items, &allocationPage) - if err != nil { - return nil, fmt.Errorf("parsing query responses: %w", err) - } - - for _, item := range allocationPage { - a, err := allocation.Decode(item.Allocation, dagcbor.Decode) - if err != nil { - return nil, fmt.Errorf("decoding data: %w", err) - } - allocations = append(allocations, a) - } - } - return allocations, nil -} - -type allocationItem struct { - Hash string `dynamodbav:"hash"` - Cause string `dynamodbav:"cause"` // note: now space DID not invocation CID - Allocation []byte `dynamodbav:"allocation"` -} - -var _ allocationstore.AllocationStore = (*DynamoAllocationStore)(nil) diff --git a/pkg/aws/dynamoprovidercontexttable.go b/pkg/aws/dynamoprovidercontexttable.go deleted file mode 100644 index 03b3ac77..00000000 --- a/pkg/aws/dynamoprovidercontexttable.go +++ /dev/null @@ -1,103 +0,0 @@ -package aws - -import ( - "context" - "errors" - "fmt" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" - "github.com/aws/aws-sdk-go-v2/service/dynamodb" - "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/storacha/go-libstoracha/ipnipublisher/store" -) - -// ErrDynamoRecordNotFound is used when there is no record in a dynamo table -// (given that GetItem does not actually error) -var ErrDynamoRecordNotFound = errors.New("no record found in dynamo table") - -// DynamoProviderContextTable implements the store.ProviderContextTable interface on dynamodb -type DynamoProviderContextTable struct { - tableName string - dynamoDbClient *dynamodb.Client -} - -var _ store.ProviderContextTable = (*DynamoProviderContextTable)(nil) - -// NewDynamoProviderContextTable returns a ProviderContextTable connected to a AWS DynamoDB table -func NewDynamoProviderContextTable(cfg aws.Config, tableName string, opts ...func(*dynamodb.Options)) *DynamoProviderContextTable { - return &DynamoProviderContextTable{ - tableName: tableName, - dynamoDbClient: dynamodb.NewFromConfig(cfg, opts...), - } -} - -// Delete implements store.ProviderContextTable. -func (d *DynamoProviderContextTable) Delete(ctx context.Context, p peer.ID, contextID []byte) error { - providerContextItem := providerContextItem{p.String(), contextID, nil} - _, err := d.dynamoDbClient.DeleteItem(ctx, &dynamodb.DeleteItemInput{ - TableName: aws.String(d.tableName), Key: providerContextItem.GetKey(), - }) - return err -} - -// Get implements store.ProviderContextTable. -func (d *DynamoProviderContextTable) Get(ctx context.Context, p peer.ID, contextID []byte) ([]byte, error) { - providerContextItem := providerContextItem{p.String(), contextID, nil} - response, err := d.dynamoDbClient.GetItem(ctx, &dynamodb.GetItemInput{ - Key: providerContextItem.GetKey(), - TableName: aws.String(d.tableName), - ProjectionExpression: aws.String("contents"), - }) - if err != nil { - return nil, fmt.Errorf("retrieving item: %w", err) - } - if response.Item == nil { - return nil, store.NewErrNotFound(ErrDynamoRecordNotFound) - } - err = attributevalue.UnmarshalMap(response.Item, &providerContextItem) - if err != nil { - return nil, fmt.Errorf("deserializing item: %w", err) - } - return providerContextItem.Contents, nil -} - -// Put implements store.ProviderContextTable. -func (d *DynamoProviderContextTable) Put(ctx context.Context, p peer.ID, contextID []byte, data []byte) error { - item, err := attributevalue.MarshalMap(providerContextItem{ - Provider: p.String(), - ContextID: contextID, - Contents: data, - }) - if err != nil { - return fmt.Errorf("serializing item: %w", err) - } - _, err = d.dynamoDbClient.PutItem(ctx, &dynamodb.PutItemInput{ - TableName: aws.String(d.tableName), Item: item, - }) - if err != nil { - return fmt.Errorf("storing item: %w", err) - } - return nil -} - -type providerContextItem struct { - Provider string `dynamodbav:"provider"` - ContextID []byte `dynamodbav:"contextID"` - Contents []byte `dynamodbav:"contents"` -} - -// GetKey returns the composite primary key of the provider & contextID in a format that can be -// sent to DynamoDB. -func (p providerContextItem) GetKey() map[string]types.AttributeValue { - provider, err := attributevalue.Marshal(p.Provider) - if err != nil { - panic(err) - } - contextID, err := attributevalue.Marshal(p.ContextID) - if err != nil { - panic(err) - } - return map[string]types.AttributeValue{"provider": provider, "contextID": contextID} -} diff --git a/pkg/aws/dynamoranlinkindex.go b/pkg/aws/dynamoranlinkindex.go deleted file mode 100644 index ae8b34e8..00000000 --- a/pkg/aws/dynamoranlinkindex.go +++ /dev/null @@ -1,90 +0,0 @@ -package aws - -import ( - "context" - "fmt" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" - "github.com/aws/aws-sdk-go-v2/service/dynamodb" - "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime/datamodel" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/storacha/go-libstoracha/ipnipublisher/store" - "github.com/storacha/piri/pkg/store/receiptstore" -) - -// DynamoRanLinkIndex implements the store.ProviderContextTable interface on dynamodb -type DynamoRanLinkIndex struct { - tableName string - dynamoDbClient *dynamodb.Client -} - -var _ receiptstore.RanLinkIndex = (*DynamoRanLinkIndex)(nil) - -// NewDynamoRanLinkIndex returns a ProviderContextTable connected to a AWS DynamoDB table -func NewDynamoRanLinkIndex(cfg aws.Config, tableName string, opts ...func(*dynamodb.Options)) *DynamoRanLinkIndex { - return &DynamoRanLinkIndex{ - tableName: tableName, - dynamoDbClient: dynamodb.NewFromConfig(cfg, opts...), - } -} - -// Get implements store.ProviderContextTable. -func (d *DynamoRanLinkIndex) Get(ctx context.Context, ran datamodel.Link) (datamodel.Link, error) { - ranLinkItem := ranLinkItem{Ran: ran.String()} - response, err := d.dynamoDbClient.GetItem(ctx, &dynamodb.GetItemInput{ - Key: ranLinkItem.GetKey(), - TableName: aws.String(d.tableName), - ProjectionExpression: aws.String("link"), - }) - if err != nil { - return nil, fmt.Errorf("retrieving item: %w", err) - } - if response.Item == nil { - return nil, store.NewErrNotFound(ErrDynamoRecordNotFound) - } - err = attributevalue.UnmarshalMap(response.Item, &ranLinkItem) - if err != nil { - return nil, fmt.Errorf("deserializing item: %w", err) - } - cid, err := cid.Decode(ranLinkItem.Link) - if err != nil { - return nil, fmt.Errorf("decoding link: %w", err) - } - return cidlink.Link{Cid: cid}, nil -} - -// Put implements store.ProviderContextTable. -func (d *DynamoRanLinkIndex) Put(ctx context.Context, ran datamodel.Link, link datamodel.Link) error { - item, err := attributevalue.MarshalMap(ranLinkItem{ - Ran: ran.String(), - Link: link.String(), - }) - if err != nil { - return fmt.Errorf("serializing item: %w", err) - } - _, err = d.dynamoDbClient.PutItem(ctx, &dynamodb.PutItemInput{ - TableName: aws.String(d.tableName), Item: item, - }) - if err != nil { - return fmt.Errorf("storing item: %w", err) - } - return nil -} - -type ranLinkItem struct { - Ran string `dynamodbav:"ran"` - Link string `dynamodbav:"link"` -} - -// GetKey returns the composite primary key of the provider & contextID in a format that can be -// sent to DynamoDB. -func (p ranLinkItem) GetKey() map[string]types.AttributeValue { - ran, err := attributevalue.Marshal(p.Ran) - if err != nil { - panic(err) - } - return map[string]types.AttributeValue{"ran": ran} -} diff --git a/pkg/aws/handlercontext.go b/pkg/aws/handlercontext.go deleted file mode 100644 index 248768e3..00000000 --- a/pkg/aws/handlercontext.go +++ /dev/null @@ -1,40 +0,0 @@ -package aws - -import ( - "io" - "net/http" - - "github.com/labstack/echo/v4" - "github.com/storacha/piri/pkg/server/handler" -) - -type HandlerContext struct { - response *echo.Response - request *http.Request -} - -func (c *HandlerContext) Request() *http.Request { - return c.request -} - -func (c *HandlerContext) Response() *echo.Response { - return c.response -} - -func (c *HandlerContext) Stream(code int, contentType string, r io.Reader) error { - header := c.Response().Header() - if header.Get(echo.HeaderContentType) == "" { - header.Set(echo.HeaderContentType, contentType) - } - c.response.WriteHeader(code) - _, err := io.Copy(c.response, r) - return err -} - -var _ handler.Context = (*HandlerContext)(nil) - -// NewHandlerContext creates a new context that satisfies [server.RequestContext] -// and allows echo style handlers to be used with AWS lambda. -func NewHandlerContext(w http.ResponseWriter, r *http.Request) *HandlerContext { - return &HandlerContext{echo.NewResponse(w, nil), r} -} diff --git a/pkg/aws/s3blobstore.go b/pkg/aws/s3blobstore.go deleted file mode 100644 index 97c4b7dd..00000000 --- a/pkg/aws/s3blobstore.go +++ /dev/null @@ -1,191 +0,0 @@ -package aws - -import ( - "context" - "encoding/base64" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "strconv" - "strings" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/multiformats/go-multicodec" - multihash "github.com/multiformats/go-multihash" - "github.com/storacha/go-libstoracha/digestutil" - "github.com/storacha/piri/pkg/presigner" - "github.com/storacha/piri/pkg/store" - "github.com/storacha/piri/pkg/store/blobstore" -) - -type KeyFormatterFunc func(digest multihash.Multihash) string - -// S3BlobStore implements the blobstore.BlobStore interface on S3 -type S3BlobStore struct { - bucket string - formatKey KeyFormatterFunc - s3Client *s3.Client -} - -var _ blobstore.Blobstore = (*S3BlobStore)(nil) - -// NewPatternKeyFormatter creates a key formatter which replaces instances of -// "{blob}" in the provided pattern with the base58btc encoding of the multihash -// digest. -func NewPatternKeyFormatter(pattern string) KeyFormatterFunc { - return func(digest multihash.Multihash) string { - return strings.ReplaceAll(pattern, "{blob}", digestutil.Format(digest)) - } -} - -func NewS3BlobStore(cfg aws.Config, bucket string, formatKey KeyFormatterFunc, opts ...func(*s3.Options)) *S3BlobStore { - if formatKey == nil { - formatKey = digestutil.Format - } - return &S3BlobStore{ - s3Client: s3.NewFromConfig(cfg, opts...), - bucket: bucket, - formatKey: formatKey, - } -} - -var _ blobstore.Object = (*s3BlobObject)(nil) - -type S3BlobPresigner struct { - bs *S3BlobStore - presignClient *s3.PresignClient -} - -// SignUploadURL implements presigner.RequestPresigner. -func (s *S3BlobPresigner) SignUploadURL(ctx context.Context, digest multihash.Multihash, size uint64, ttl uint64) (url.URL, http.Header, error) { - digestInfo, err := multihash.Decode(digest) - if err != nil { - return url.URL{}, nil, fmt.Errorf("decoding digest: %w", err) - } - if digestInfo.Code != uint64(multicodec.Sha2_256) { - return url.URL{}, nil, fmt.Errorf("unsupported digest: %d", digestInfo.Code) - } - - signedReq, err := s.presignClient.PresignPutObject( - ctx, - &s3.PutObjectInput{ - Bucket: aws.String(s.bs.bucket), - Key: aws.String(s.bs.formatKey(digest)), - ContentLength: aws.Int64(int64(size)), - ChecksumSHA256: aws.String(base64.StdEncoding.EncodeToString(digestInfo.Digest)), - }, - s3.WithPresignExpires(time.Duration(int64(ttl)*int64(time.Second))), - ) - if err != nil { - return url.URL{}, nil, fmt.Errorf("signing request: %w", err) - } - - reqURL, err := url.Parse(signedReq.URL) - if err != nil { - return url.URL{}, nil, fmt.Errorf("parsing signed URL: %w", err) - } - - return *reqURL, signedReq.SignedHeader, nil -} - -// VerifyUploadURL implements presigner.RequestPresigner. -func (s *S3BlobPresigner) VerifyUploadURL(ctx context.Context, url url.URL, headers http.Header) (url.URL, http.Header, error) { - panic("unimplemented") -} - -var _ presigner.RequestPresigner = (*S3BlobPresigner)(nil) - -func (s *S3BlobStore) PresignClient() presigner.RequestPresigner { - presignClient := s3.NewPresignClient(s.s3Client, func(opt *s3.PresignOptions) { - opt.Presigner = v4.NewSigner(func(so *v4.SignerOptions) { - o := s.s3Client.Options() - so.Logger = o.Logger - so.LogSigning = o.ClientLogMode.IsSigning() - so.DisableURIPathEscaping = true - // This is the magic sauce which makes SHA256 checksums work. - // It causes the X-Amz-Sdk-Checksum-Algorithm, and X-Amz-Checksum-Sha256 - // to be included as HTTP headers instead of query parameters in the url. - // The S3 backend currently silently ignores these if they are sent as - // query parameters. - so.DisableHeaderHoisting = true - }) - }) - return &S3BlobPresigner{s, presignClient} -} - -// Put implements blobstore.Blobstore. -func (s *S3BlobStore) Put(ctx context.Context, digest multihash.Multihash, size uint64, body io.Reader) error { - digestInfo, err := multihash.Decode(digest) - if err != nil { - return fmt.Errorf("decoding digest: %w", err) - } - if digestInfo.Code != uint64(multicodec.Sha2_256) { - return fmt.Errorf("unsupported digest: %d", digestInfo.Code) - } - _, err = s.s3Client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(s.formatKey(digest)), - Body: body, - ContentLength: aws.Int64(int64(size)), - ChecksumSHA256: aws.String(base64.StdEncoding.EncodeToString(digestInfo.Digest)), - }) - return err -} - -// Get implements blobstore.Blobstore. -func (s *S3BlobStore) Get(ctx context.Context, digest multihash.Multihash, opts ...blobstore.GetOption) (blobstore.Object, error) { - config := blobstore.NewGetConfig() - config.ProcessOptions(opts) - - var rangeParam *string - if config.Range().Start != 0 || config.Range().End != nil { - rangeString := fmt.Sprintf("bytes=%d-", config.Range().Start) - if config.Range().End != nil { - rangeString += strconv.FormatUint(*config.Range().End, 10) - } - rangeParam = &rangeString - } - outPut, err := s.s3Client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(s.formatKey(digest)), - Range: rangeParam, - }) - if err != nil { - var noSuchKeyError *types.NoSuchKey - // wrap in error recognizable as a not found error for Store interface consumers - if errors.As(err, &noSuchKeyError) { - return nil, store.ErrNotFound - } - return nil, err - } - return &s3BlobObject{outPut}, nil -} - -type s3BlobObject struct { - outPut *s3.GetObjectOutput -} - -// Body implements blobstore.Object. -func (s *s3BlobObject) Body() io.ReadCloser { - return s.outPut.Body -} - -// Size implements blobstore.Object. -func (s *s3BlobObject) Size() int64 { - return *s.outPut.ContentLength -} - -// Delete implements blobstore.Blobstore. -func (s *S3BlobStore) Delete(ctx context.Context, digest multihash.Multihash) error { - _, err := s.s3Client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(s.formatKey(digest)), - }) - return err -} diff --git a/pkg/aws/s3indexerproofs.go b/pkg/aws/s3indexerproofs.go deleted file mode 100644 index 7955b59c..00000000 --- a/pkg/aws/s3indexerproofs.go +++ /dev/null @@ -1,58 +0,0 @@ -package aws - -import ( - "context" - "fmt" - "io" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/storacha/go-ucanto/core/delegation" -) - -// S3IndexerProofs returns delegation proofs for the indexer -type S3IndexerProofs struct { - bucket string - keyPrefix string - s3Client *s3.Client -} - -// Get implements store.Store. -func (s *S3IndexerProofs) Get(ctx context.Context) ([]delegation.Proof, error) { - list, err := s.s3Client.ListObjects(ctx, &s3.ListObjectsInput{ - Bucket: aws.String(s.bucket), - Prefix: aws.String(s.keyPrefix), - }) - if err != nil { - return nil, fmt.Errorf("listing objects: %w", err) - } - proofs := make([]delegation.Proof, 0, len(list.Contents)) - for _, obj := range list.Contents { - - outPut, err := s.s3Client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucket), - Key: obj.Key, - }) - if err != nil { - return nil, fmt.Errorf("fetching proof from S3: %w", err) - } - data, err := io.ReadAll(outPut.Body) - if err != nil { - return nil, fmt.Errorf("reading proof data: %w", err) - } - d, err := delegation.Extract(data) - if err != nil { - return nil, fmt.Errorf("decoding proof: %w", err) - } - proofs = append(proofs, delegation.FromDelegation(d)) - } - return proofs, nil -} - -func NewS3IndexerProofs(cfg aws.Config, bucket string, keyPrefix string, opts ...func(*s3.Options)) *S3IndexerProofs { - return &S3IndexerProofs{ - s3Client: s3.NewFromConfig(cfg, opts...), - bucket: bucket, - keyPrefix: keyPrefix, - } -} diff --git a/pkg/aws/s3store.go b/pkg/aws/s3store.go deleted file mode 100644 index 7edcaca4..00000000 --- a/pkg/aws/s3store.go +++ /dev/null @@ -1,113 +0,0 @@ -package aws - -import ( - "context" - "crypto/md5" - "encoding/hex" - "errors" - "fmt" - "io" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/aws/smithy-go" - "github.com/storacha/go-libstoracha/ipnipublisher/store" -) - -// S3Store implements the store.Store interface on S3 -type S3Store struct { - bucket string - keyPrefix string - s3Client *s3.Client -} - -var _ store.Store = (*S3Store)(nil) - -// Get implements store.Store. -func (s *S3Store) Get(ctx context.Context, key string) (io.ReadCloser, error) { - outPut, err := s.s3Client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(s.keyPrefix + key), - }) - if err != nil { - var noSuchKeyError *types.NoSuchKey - // wrap in error recognizable as a not found error for Store interface consumers - if errors.As(err, &noSuchKeyError) { - return nil, store.NewErrNotFound(err) - } - return nil, err - } - return outPut.Body, nil -} - -// Put implements store.Store. -func (s *S3Store) Put(ctx context.Context, key string, length uint64, data io.Reader) error { - _, err := s.s3Client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(s.keyPrefix + key), - Body: data, - ContentLength: aws.Int64(int64(length)), - }) - return err -} - -func (s *S3Store) Replace(ctx context.Context, key string, old io.Reader, length uint64, new io.Reader) error { - input := s3.PutObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(s.keyPrefix + key), - Body: new, - ContentLength: aws.Int64(int64(length)), - } - - // use conditional write requests to replace the data - if old == nil { - input.IfNoneMatch = aws.String("*") - } else { - b, err := io.ReadAll(old) - if err != nil { - return err - } - md5hash := md5.Sum(b) - etag := fmt.Sprintf("%q", hex.EncodeToString(md5hash[:])) - input.IfMatch = aws.String(etag) - } - - _, err := s.s3Client.PutObject(ctx, &input) - if err != nil { - var oe smithy.APIError - if errors.As(err, &oe) { - // This method is used by the IPNI publisher to write a new head to the - // chain. We can receive one of the following error code because we're - // using `If-Match`. - // - // PreconditionFailed: At least one of the preconditions you specified did - // not hold. - // - // OperationAborted: A conflicting conditional action is currently in - // progress against this resource. Try again. - // - // If we receive OperationAborted then we'd get PreconditionFailed on the - // next try, since we don't put the same head twice, and so there is no - // chance to succeed with the same If-Match Etag, since the content will - // have changed due to the "conflicting conditional action" already in - // progress. Hence we don't try again and simply return the error - // [store.ErrPreconditionFailed] so that the calling code can try again. - // - // When ErrPreconditionFailed is returned, a new advert must be - // constructed that references the new head and the operation retried. - if oe.ErrorCode() == "PreconditionFailed" || oe.ErrorCode() == "OperationAborted" { - return store.ErrPreconditionFailed - } - } - } - return err -} - -func NewS3StoreWithClient(client *s3.Client, bucket string, keyPrefix string) *S3Store { - return &S3Store{s3Client: client, bucket: bucket, keyPrefix: keyPrefix} -} - -func NewS3Store(cfg aws.Config, bucket string, keyPrefix string, opts ...func(*s3.Options)) *S3Store { - return NewS3StoreWithClient(s3.NewFromConfig(cfg, opts...), bucket, keyPrefix) -} diff --git a/pkg/aws/s3store_test.go b/pkg/aws/s3store_test.go deleted file mode 100644 index 51867932..00000000 --- a/pkg/aws/s3store_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package aws_test - -import ( - "bytes" - "encoding/hex" - "net/url" - "runtime" - "testing" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/storacha/go-libstoracha/ipnipublisher/store" - "github.com/storacha/go-libstoracha/testutil" - paws "github.com/storacha/piri/pkg/aws" - piritutil "github.com/storacha/piri/pkg/internal/testutil" - "github.com/stretchr/testify/require" -) - -func TestS3StoreReplace(t *testing.T) { - // This test expects docker to be running in linux CI environments and fails if it's not - if piritutil.IsRunningInCI(t) && runtime.GOOS == "linux" { - if !piritutil.IsDockerAvailable(t) { - t.Fatalf("docker is expected in CI linux testing environments, but wasn't found") - } - } - // otherwise this test is running locally, skip it if docker isn't available - if !piritutil.IsDockerAvailable(t) { - t.SkipNow() - } - - endpoint := piritutil.StartMinioContainer(t) - client := newS3Client(t, testutil.Must(url.Parse("http://"+endpoint))(t)) - - bucketName := hex.EncodeToString(testutil.RandomBytes(t, 16)) - createBucket(t, client, bucketName) - - st := paws.NewS3StoreWithClient(client, bucketName, "") - - t.Run("conditional write", func(t *testing.T) { - key := hex.EncodeToString(testutil.RandomBytes(t, 4)) - first := testutil.RandomBytes(t, 32) - second := testutil.RandomBytes(t, 32) - third := testutil.RandomBytes(t, 32) - - err := st.Put(t.Context(), key, 32, bytes.NewReader(first)) - require.NoError(t, err) - - err = st.Replace(t.Context(), key, bytes.NewReader(first), 32, bytes.NewReader(second)) - require.NoError(t, err) - - err = st.Replace(t.Context(), key, bytes.NewReader(first), 32, bytes.NewReader(third)) - require.ErrorIs(t, err, store.ErrPreconditionFailed) - }) - - t.Run("conditional first write", func(t *testing.T) { - key := hex.EncodeToString(testutil.RandomBytes(t, 4)) - first := testutil.RandomBytes(t, 32) - - err := st.Replace(t.Context(), key, nil, 32, bytes.NewReader(first)) - require.NoError(t, err) - }) -} - -func newS3Client(t *testing.T, endpoint *url.URL) *s3.Client { - cfg, err := config.LoadDefaultConfig( - t.Context(), - config.WithCredentialsProvider(credentials.StaticCredentialsProvider{ - Value: aws.Credentials{ - AccessKeyID: "minioadmin", - SecretAccessKey: "minioadmin", - }, - }), - func(o *config.LoadOptions) error { - o.Region = "us-east-1" - return nil - }, - ) - require.NoError(t, err) - - return s3.NewFromConfig(cfg, func(o *s3.Options) { - base := endpoint.String() - o.BaseEndpoint = &base - o.UsePathStyle = true - }) -} - -func createBucket(t *testing.T, client *s3.Client, name string) { - _, err := client.CreateBucket(t.Context(), &s3.CreateBucketInput{Bucket: aws.String(name)}) - require.NoError(t, err) -} diff --git a/pkg/aws/service.go b/pkg/aws/service.go deleted file mode 100644 index 5bfe9f90..00000000 --- a/pkg/aws/service.go +++ /dev/null @@ -1,389 +0,0 @@ -package aws - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "maps" - "net/url" - "os" - "strings" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/dynamodb" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/ssm" - "github.com/ipni/go-libipni/maurl" - "github.com/multiformats/go-multiaddr" - publisherqueue "github.com/storacha/go-libstoracha/ipnipublisher/queue" - awspublisherqueue "github.com/storacha/go-libstoracha/ipnipublisher/queue/aws" - "github.com/storacha/go-libstoracha/ipnipublisher/store" - "github.com/storacha/go-libstoracha/metadata" - "github.com/storacha/go-ucanto/client" - "github.com/storacha/go-ucanto/core/delegation" - "github.com/storacha/go-ucanto/did" - "github.com/storacha/go-ucanto/principal" - ed25519 "github.com/storacha/go-ucanto/principal/ed25519/signer" - edverifier "github.com/storacha/go-ucanto/principal/ed25519/verifier" - "github.com/storacha/go-ucanto/principal/signer" - ucanhttp "github.com/storacha/go-ucanto/transport/http" - "github.com/storacha/go-ucanto/validator" - - "github.com/storacha/piri/pkg/access" - "github.com/storacha/piri/pkg/presets" - "github.com/storacha/piri/pkg/principalresolver" - "github.com/storacha/piri/pkg/service/storage" - "github.com/storacha/piri/pkg/store/delegationstore" - "github.com/storacha/piri/pkg/store/receiptstore" -) - -// ErrMissingSecret means that the value returned from Secrets was empty -var ErrMissingSecret = errors.New("missing value for secret") - -func mustGetEnv(envVar string) string { - value := os.Getenv(envVar) - if len(value) == 0 { - panic(fmt.Errorf("missing env var: %s", envVar)) - } - return value -} - -var ErrIndexingServiceProofsMissing = errors.New("indexing service proofs are missing") - -type Config struct { - Config aws.Config - S3Options []func(*s3.Options) - DynamoOptions []func(*dynamodb.Options) - SentryDSN string - SentryEnvironment string - AllocationsTableName string - AcceptanceTableName string - BlobStoreBucketEndpoint string - BlobStoreBucketRegion string - BlobStoreBucketAccessKeyID string - BlobStoreBucketSecretAccessKey string - BlobStoreBucketKeyPattern string - BlobStoreBucket string - AggregatesBucket string - AggregatesPrefix string - BufferBucket string - BufferPrefix string - ChunkLinksTableName string - MetadataTableName string - IPNIStoreBucket string - IPNIStorePrefix string - IPNIAnnounceURLs []url.URL - ClaimStoreBucket string - ClaimStorePrefix string - PublicURL string - IndexingServiceDID string - IndexingServiceURL string - IndexingServiceProof string - UploadServiceDID did.DID - UploadServiceURL *url.URL - IPNIPublisherAnnounceAddress string - BlobsPublicURL string - RanLinkIndexTableName string - ReceiptStoreBucket string - ReceiptStorePrefix string - SQSPublishingQueueID string - SQSAdvertisementPublishingQueueID string - PublishingBucket string - PrincipalMapping map[string]string - principal.Signer -} - -func mustGetSSMParams(ctx context.Context, client *ssm.Client, names ...string) map[string]string { - response, err := client.GetParameters(ctx, &ssm.GetParametersInput{ - Names: names, - WithDecryption: aws.Bool(true), - }) - if err != nil { - panic(fmt.Errorf("retrieving SSM parameters: %w", err)) - } - params := map[string]string{} - for _, name := range names { - value := "" - for _, p := range response.Parameters { - if *p.Name == name { - value = *p.Value - break - } - } - if value == "" { - panic(ErrMissingSecret) - } - params[name] = value - } - return params -} - -// FromEnv constructs the AWS Configuration from the environment -func FromEnv(ctx context.Context) Config { - awsConfig, err := config.LoadDefaultConfig(ctx) - if err != nil { - panic(fmt.Errorf("loading aws default config: %w", err)) - } - - ssmClient := ssm.NewFromConfig(awsConfig) - secretNames := []string{mustGetEnv("PRIVATE_KEY")} - for _, n := range []string{ - "BLOB_STORE_BUCKET_ACCESS_KEY_ID", - "BLOB_STORE_BUCKET_SECRET_ACCESS_KEY", - } { - if os.Getenv(n) != "" { - secretNames = append(secretNames, os.Getenv(n)) - } - } - secrets := mustGetSSMParams(ctx, ssmClient, secretNames...) - - id, err := ed25519.Parse(secrets[mustGetEnv("PRIVATE_KEY")]) - if err != nil { - panic(fmt.Errorf("parsing private key: %s", err)) - } - - if len(os.Getenv("DID")) != 0 { - d, err := did.Parse(os.Getenv("DID")) - if err != nil { - panic(fmt.Errorf("parsing DID: %w", err)) - } - id, err = signer.Wrap(id, d) - if err != nil { - panic(fmt.Errorf("wrapping server DID: %w", err)) - } - } - - ipniStoreKeyPrefix := os.Getenv("IPNI_STORE_KEY_PREFIX") - if len(ipniStoreKeyPrefix) == 0 { - ipniStoreKeyPrefix = "ipni/v1/ad/" - } - - ipniPublisherAnnounceAddress := fmt.Sprintf("/dns/%s/https", mustGetEnv("IPNI_STORE_BUCKET_REGIONAL_DOMAIN")) - - network, err := presets.ParseNetwork(mustGetEnv("PIRI_NETWORK")) - if err != nil { - panic(fmt.Errorf("invalid network: %w. Valid values are: %q", err, presets.AvailableNetworks)) - } - - preset, err := presets.GetPreset(network) - if err != nil { - panic(fmt.Errorf("invalid network: %w", err)) - } - - blobsPublicURL := "https://" + mustGetEnv("BLOB_STORE_BUCKET_REGIONAL_DOMAIN") - var principalMapping map[string]string - if os.Getenv("PRINCIPAL_MAPPING") != "" { - principalMapping = map[string]string{} - maps.Copy(principalMapping, preset.Services.PrincipalMapping) - var pm map[string]string - err := json.Unmarshal([]byte(os.Getenv("PRINCIPAL_MAPPING")), &pm) - if err != nil { - panic(fmt.Errorf("parsing principal mapping: %w", err)) - } - maps.Copy(principalMapping, pm) - } else { - principalMapping = preset.Services.PrincipalMapping - } - - var ipniAnnounceURLs []url.URL - if os.Getenv("IPNI_ANNOUNCE_URLS") != "" { - var urls []string - err := json.Unmarshal([]byte(os.Getenv("IPNI_ANNOUNCE_URLS")), &urls) - if err != nil { - panic(fmt.Errorf("parsing IPNI announce URLs JSON: %w", err)) - } - for _, s := range urls { - url, err := url.Parse(s) - if err != nil { - panic(fmt.Errorf("parsing IPNI announce URL: %s: %w", s, err)) - } - ipniAnnounceURLs = append(ipniAnnounceURLs, *url) - } - } else { - ipniAnnounceURLs = preset.Services.IPNIAnnounceURLs - } - - return Config{ - Config: awsConfig, - SentryDSN: os.Getenv("SENTRY_DSN"), - SentryEnvironment: os.Getenv("SENTRY_ENVIRONMENT"), - Signer: id, - ChunkLinksTableName: mustGetEnv("CHUNK_LINKS_TABLE_NAME"), - MetadataTableName: mustGetEnv("METADATA_TABLE_NAME"), - IPNIStoreBucket: mustGetEnv("IPNI_STORE_BUCKET_NAME"), - IPNIStorePrefix: ipniStoreKeyPrefix, - IPNIPublisherAnnounceAddress: ipniPublisherAnnounceAddress, - IPNIAnnounceURLs: ipniAnnounceURLs, - BlobsPublicURL: blobsPublicURL, - ClaimStoreBucket: mustGetEnv("CLAIM_STORE_BUCKET_NAME"), - ClaimStorePrefix: os.Getenv("CLAIM_STORE_KEY_REFIX"), - AllocationsTableName: mustGetEnv("ALLOCATIONS_TABLE_NAME"), - AcceptanceTableName: mustGetEnv("ACCEPTANCE_TABLE_NAME"), - BlobStoreBucketEndpoint: os.Getenv("BLOB_STORE_BUCKET_ENDPOINT"), - BlobStoreBucketRegion: os.Getenv("BLOB_STORE_BUCKET_REGION"), - BlobStoreBucketAccessKeyID: secrets[os.Getenv("BLOB_STORE_BUCKET_ACCESS_KEY_ID")], - BlobStoreBucketSecretAccessKey: secrets[os.Getenv("BLOB_STORE_BUCKET_SECRET_ACCESS_KEY")], - BlobStoreBucketKeyPattern: os.Getenv("BLOB_STORE_BUCKET_KEY_PATTERN"), - BlobStoreBucket: mustGetEnv("BLOB_STORE_BUCKET_NAME"), - BufferBucket: os.Getenv("BUFFER_BUCKET_NAME"), - BufferPrefix: os.Getenv("BUFFER_KEY_PREFIX"), - AggregatesBucket: os.Getenv("AGGREGATES_BUCKET_NAME"), - AggregatesPrefix: os.Getenv("AGGREGATES_KEY_PREFIX"), - PublicURL: mustGetEnv("PUBLIC_URL"), - IndexingServiceDID: mustGetEnv("INDEXING_SERVICE_DID"), - IndexingServiceURL: mustGetEnv("INDEXING_SERVICE_URL"), - IndexingServiceProof: mustGetEnv("INDEXING_SERVICE_PROOF"), - UploadServiceDID: preset.Services.UploadServiceDID, - UploadServiceURL: preset.Services.UploadServiceURL, - RanLinkIndexTableName: mustGetEnv("RAN_LINK_INDEX_TABLE_NAME"), - ReceiptStoreBucket: mustGetEnv("RECEIPT_STORE_BUCKET_NAME"), - ReceiptStorePrefix: os.Getenv("RECEIPT_STORE_KEY_PREFIX"), - SQSPublishingQueueID: mustGetEnv("IPNI_PUBLISHER_QUEUE_ID"), - PublishingBucket: mustGetEnv("IPNI_PUBLISHER_BUCKET_NAME"), - SQSAdvertisementPublishingQueueID: mustGetEnv("IPNI_ADVERTISEMENT_PUBLISHING_QUEUE_ID"), - PrincipalMapping: principalMapping, - } -} - -func Construct(cfg Config) (storage.Service, error) { - uploadServiceConn, err := client.NewConnection(cfg.UploadServiceDID, ucanhttp.NewChannel(cfg.UploadServiceURL)) - if err != nil { - return nil, fmt.Errorf("creating upload service connection: %w", err) - } - - blobStoreOpts := cfg.S3Options - if cfg.BlobStoreBucketAccessKeyID != "" && cfg.BlobStoreBucketSecretAccessKey != "" { - blobStoreOpts = append(blobStoreOpts, func(opts *s3.Options) { - opts.Region = cfg.BlobStoreBucketRegion - opts.Credentials = credentials.NewStaticCredentialsProvider( - cfg.BlobStoreBucketAccessKeyID, - cfg.BlobStoreBucketSecretAccessKey, - "", - ) - if cfg.BlobStoreBucketEndpoint != "" { - opts.BaseEndpoint = &cfg.BlobStoreBucketEndpoint - opts.UsePathStyle = true - } - }) - } - var formatKey KeyFormatterFunc - if cfg.BlobStoreBucketKeyPattern != "" { - formatKey = NewPatternKeyFormatter(cfg.BlobStoreBucketKeyPattern) - } - blobStore := NewS3BlobStore(cfg.Config, cfg.BlobStoreBucket, formatKey, blobStoreOpts...) - allocationStore := NewDynamoAllocationStore(cfg.Config, cfg.AllocationsTableName, cfg.DynamoOptions...) - acceptanceStore := NewDynamoAcceptanceStore(cfg.Config, cfg.AcceptanceTableName, cfg.DynamoOptions...) - claimStore := delegationstore.New( - NewSimpleStoreObjectAdapter(NewS3Store(cfg.Config, cfg.ClaimStoreBucket, cfg.ClaimStorePrefix, cfg.S3Options...)), - delegationstore.S3KeyEncoder{}, - ) - ipniStore := NewS3Store(cfg.Config, cfg.IPNIStoreBucket, cfg.IPNIStorePrefix, cfg.S3Options...) - chunkLinksTable := NewDynamoProviderContextTable(cfg.Config, cfg.ChunkLinksTableName, cfg.DynamoOptions...) - metadataTable := NewDynamoProviderContextTable(cfg.Config, cfg.MetadataTableName, cfg.DynamoOptions...) - publisherStore := store.NewPublisherStore(ipniStore, chunkLinksTable, metadataTable, store.WithMetadataContext(metadata.MetadataContext)) - pubURL, err := url.Parse(cfg.PublicURL) - if err != nil { - return nil, fmt.Errorf("parsing public url: %w", err) - } - blobsPublicURL, err := url.Parse(cfg.BlobsPublicURL) - if err != nil { - return nil, fmt.Errorf("parsing blob store public url: %w", err) - } - indexingServiceDID, err := did.Parse(cfg.IndexingServiceDID) - if err != nil { - return nil, fmt.Errorf("parsing indexing service did: %w", err) - } - indexingServiceURL, err := url.Parse(cfg.IndexingServiceURL) - if err != nil { - return nil, fmt.Errorf("parsing indexing service url: %w", err) - } - var indexingServiceProofs delegation.Proofs - proof, err := delegation.Parse(cfg.IndexingServiceProof) - if err != nil { - return nil, fmt.Errorf("parsing indexing service proof") - } - indexingServiceProofs = append(indexingServiceProofs, delegation.FromDelegation(proof)) - if len(indexingServiceProofs) == 0 { - return nil, ErrIndexingServiceProofsMissing - } - ranLinkIndex := NewDynamoRanLinkIndex(cfg.Config, cfg.RanLinkIndexTableName, cfg.DynamoOptions...) - s3ReceiptStore := NewS3Store(cfg.Config, cfg.ReceiptStoreBucket, cfg.ReceiptStorePrefix, cfg.S3Options...) - receiptStore := receiptstore.New( - NewSimpleStoreObjectAdapter(s3ReceiptStore), - receiptstore.S3KeyEncoder{}, - ranLinkIndex, - ) - - publishingQueue := awspublisherqueue.NewSQSPublishingQueue(cfg.Config, cfg.SQSPublishingQueueID, cfg.PublishingBucket) - queuePublisher := publisherqueue.NewQueuePublisher(publishingQueue) - - presolv, err := principalresolver.NewHTTPResolver([]did.DID{indexingServiceDID}) - if err != nil { - return nil, fmt.Errorf("creating http principal resolver: %w", err) - } - cachedpresolv, err := principalresolver.NewCachedResolver(presolv, 24*time.Hour) - if err != nil { - return nil, fmt.Errorf("creating cached principal resolver: %w", err) - } - - claimValidationCtx := validator.NewClaimContext( - cfg.Signer.Verifier(), - validator.IsSelfIssued, - func(context.Context, validator.Authorization[any]) validator.Revoked { - return nil - }, - validator.ProofUnavailable, - edverifier.Parse, - cachedpresolv.ResolveDIDKey, - validator.NotExpiredNotTooEarly, - ) - - opts := []storage.Option{ - storage.WithIdentity(cfg.Signer), - storage.WithBlobstore(blobStore), - storage.WithAllocationStore(allocationStore), - storage.WithAcceptanceStore(acceptanceStore), - storage.WithClaimStore(claimStore), - storage.WithPublisherStore(publisherStore), - storage.WithAsyncPublisher(queuePublisher), - storage.WithPublicURL(*pubURL), - storage.WithPublisherIndexingServiceConfig(indexingServiceDID, *indexingServiceURL), - storage.WithPublisherIndexingServiceProof(indexingServiceProofs...), - storage.WithReceiptStore(receiptStore), - storage.WithBlobsPublicURL(*blobsPublicURL), - storage.WithBlobsPresigner(blobStore.PresignClient()), - storage.WithClaimValidationContext(claimValidationCtx), - } - - var blobAddr multiaddr.Multiaddr - if cfg.BlobStoreBucketKeyPattern != "" { - blobPublicAddr, err := maurl.FromURL(blobsPublicURL) - if err != nil { - return nil, fmt.Errorf("parsing blobs public url to address: %w", err) - } - pathAddr, err := multiaddr.NewMultiaddr("/http-path/" + url.PathEscape(cfg.BlobStoreBucketKeyPattern)) - if err != nil { - return nil, fmt.Errorf("parsing multiaddr for blob store key pattern: %w", err) - } - blobAddr = multiaddr.Join(blobPublicAddr, pathAddr) - pattern := blobsPublicURL.String() - if strings.HasSuffix(pattern, "/") { - pattern = fmt.Sprintf("%s%s", pattern, cfg.BlobStoreBucketKeyPattern) - } else { - pattern = fmt.Sprintf("%s/%s", pattern, cfg.BlobStoreBucketKeyPattern) - } - access, err := access.NewPatternAccess(pattern) - if err != nil { - return nil, fmt.Errorf("setting up pattern acess: %w", err) - } - opts = append(opts, storage.WithBlobsAccess(access)) - } - if blobAddr != nil { - opts = append(opts, storage.WithPublisherBlobAddress(blobAddr)) - } - return storage.New(uploadServiceConn, opts...) -} diff --git a/pkg/aws/simplestoreobjectadapter.go b/pkg/aws/simplestoreobjectadapter.go deleted file mode 100644 index 84caa7b9..00000000 --- a/pkg/aws/simplestoreobjectadapter.go +++ /dev/null @@ -1,78 +0,0 @@ -package aws - -import ( - "context" - "fmt" - "io" - "iter" - - "github.com/storacha/go-libstoracha/ipnipublisher/store" - - "github.com/storacha/piri/pkg/store/objectstore" -) - -// simpleStoreObjectAdapter adapts a store.SimpleStore to objectstore.ListableStore. -// This enables use of SimpleStore backends (like AWS S3Store) with stores that -// require objectstore.ListableStore (like delegationstore). -type simpleStoreObjectAdapter struct { - store store.SimpleStore -} - -var _ objectstore.ListableStore = (*simpleStoreObjectAdapter)(nil) - -// NewSimpleStoreObjectAdapter creates an objectstore.ListableStore adapter for a SimpleStore. -func NewSimpleStoreObjectAdapter(s store.SimpleStore) objectstore.ListableStore { - return &simpleStoreObjectAdapter{store: s} -} - -func (a *simpleStoreObjectAdapter) Put(ctx context.Context, key string, size uint64, data io.Reader) error { - return a.store.Put(ctx, key, size, data) -} - -func (a *simpleStoreObjectAdapter) Get(ctx context.Context, key string, opts ...objectstore.GetOption) (objectstore.Object, error) { - // Process options but SimpleStore doesn't support range requests - cfg := objectstore.NewGetConfig() - cfg.ProcessOptions(opts) - r := cfg.Range() - if r.Start != 0 || r.End != nil { - return nil, fmt.Errorf("SimpleStore adapter does not support range requests") - } - - body, err := a.store.Get(ctx, key) - if err != nil { - if store.IsNotFound(err) { - return nil, objectstore.ErrNotExist - } - return nil, err - } - - return &simpleStoreObject{body: body}, nil -} - -func (a *simpleStoreObjectAdapter) Delete(ctx context.Context, key string) error { - return fmt.Errorf("SimpleStore adapter does not support Delete") -} - -func (a *simpleStoreObjectAdapter) Exists(ctx context.Context, key string) (bool, error) { - return false, fmt.Errorf("SimpleStore adapter does not support Exists") -} - -func (a *simpleStoreObjectAdapter) ListPrefix(ctx context.Context, prefix string) iter.Seq2[string, error] { - return func(yield func(string, error) bool) { - yield("", fmt.Errorf("SimpleStore adapter does not support ListPrefix")) - } -} - -// simpleStoreObject wraps an io.ReadCloser as objectstore.Object. -type simpleStoreObject struct { - body io.ReadCloser -} - -func (o *simpleStoreObject) Size() int64 { - // SimpleStore doesn't provide size information - return -1 -} - -func (o *simpleStoreObject) Body() io.ReadCloser { - return o.body -} diff --git a/pkg/fx/claims/provider.go b/pkg/fx/claims/provider.go index 828cffc8..50a141f1 100644 --- a/pkg/fx/claims/provider.go +++ b/pkg/fx/claims/provider.go @@ -27,5 +27,5 @@ func NewService( claimStore claimstore.ClaimStore, pub publisherSvc.Publisher, ) *claims.ClaimService { - return claims.NewV2(claimStore, pub) + return claims.New(claimStore, pub) } diff --git a/pkg/pdp/scheduler/utils.go b/pkg/pdp/scheduler/utils.go deleted file mode 100644 index 9cdc06af..00000000 --- a/pkg/pdp/scheduler/utils.go +++ /dev/null @@ -1,39 +0,0 @@ -package scheduler - -import ( - "sync" - "time" -) - -// Every is a helper function that will call the provided callback -// function at most once every `passEvery` duration. If the function is called -// more frequently than that, it will return nil and not call the callback. -// Deprecated: Use NewPeriodicScheduler instead. -func Every[P, R any](passInterval time.Duration, cb func(P) R) func(P) R { - var lastCall time.Time - var lk sync.Mutex - - return func(param P) R { - lk.Lock() - defer lk.Unlock() - - if time.Since(lastCall) < passInterval { - return *new(R) - } - - defer func() { - lastCall = time.Now() - }() - return cb(param) - } -} - -// NewPeriodicScheduler creates a new PeriodicScheduler with the given interval and runner function. -// This is a helper function to make it easier to create a PeriodicScheduler for tasks -// that need to run periodically. -func NewPeriodicScheduler(interval time.Duration, runner func(AddTaskFunc) error) *PeriodicScheduler { - return &PeriodicScheduler{ - Interval: interval, - Runner: runner, - } -} diff --git a/pkg/pdp/testing/chain_client.go b/pkg/pdp/testing/chain_client.go deleted file mode 100644 index 79904487..00000000 --- a/pkg/pdp/testing/chain_client.go +++ /dev/null @@ -1,166 +0,0 @@ -package testing - -import ( - "context" - crand "crypto/rand" - "sync" - "testing" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/store" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - mh "github.com/multiformats/go-multihash" - "github.com/stretchr/testify/require" -) - -type FakeChainClient struct { - currentMu sync.Mutex - currentHeight abi.ChainEpoch - currentTipSet *types.TipSet - notifyChans []chan []*api.HeadChange - - miner address.Address - t testing.TB -} - -func RandomBytes(t testing.TB, size int) []byte { - bytes := make([]byte, size) - _, err := crand.Read(bytes) - require.NoError(t, err) - return bytes -} - -func RandomCID(t testing.TB) cid.Cid { - bytes := RandomBytes(t, 10) - c, err := cid.Prefix{ - Version: 1, - Codec: cid.Raw, - MhType: mh.SHA2_256, - MhLength: -1, - }.Sum(bytes) - require.NoError(t, err) - return c -} - -func NewFakeChainClient(t testing.TB) *FakeChainClient { - // Create a fake tipset at height 1 - miner, err := address.NewIDAddress(1) - require.NoError(t, err) - - ts, err := types.NewTipSet([]*types.BlockHeader{ - { - Height: 1, - Miner: miner, - Parents: []cid.Cid{RandomCID(t)}, - ParentStateRoot: RandomCID(t), - ParentMessageReceipts: RandomCID(t), - Messages: RandomCID(t), - }, - }) - require.NoError(t, err) - - return &FakeChainClient{ - currentHeight: 0, - currentTipSet: ts, - notifyChans: make([]chan []*api.HeadChange, 0), - miner: miner, - t: t, - } -} - -func (c *FakeChainClient) CurrentHeight() abi.ChainEpoch { - c.currentMu.Lock() - defer c.currentMu.Unlock() - return c.currentHeight -} - -func (c *FakeChainClient) ChainHead(ctx context.Context) (*types.TipSet, error) { - c.currentMu.Lock() - defer c.currentMu.Unlock() - - return c.currentTipSet, nil -} - -func (c *FakeChainClient) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, error) { - c.currentMu.Lock() - defer c.currentMu.Unlock() - - // Create notification channel - ch := make(chan []*api.HeadChange, 16) - c.notifyChans = append(c.notifyChans, ch) - - // Send current head as the first notification - // HCCurrent is always the first notification in the "real" implementation - // Gripe: how the F^%! Filecoin existed for so long with a testing harness that solves this is beyond me. - ch <- []*api.HeadChange{ - { - Type: store.HCCurrent, - Val: c.currentTipSet, - }, - } - - return ch, nil -} - -func (c *FakeChainClient) StateGetRandomnessDigestFromBeacon(ctx context.Context, randEpoch abi.ChainEpoch, tsk types.TipSetKey) (abi.Randomness, error) { - randBytes := make([]byte, 32) - - // Use epoch value to influence the randomness - epoch := uint64(randEpoch) - for i := 0; i < 8 && i < len(randBytes); i++ { - randBytes[i] = byte((epoch >> (i * 8)) & 0xff) - } - - return randBytes, nil -} - -func (c *FakeChainClient) AdvanceChain() abi.ChainEpoch { - return c.AdvanceByHeight(1) -} - -// AdvanceHeight advances the chain by the specified number of epochs -func (c *FakeChainClient) AdvanceByHeight(epochs int64) abi.ChainEpoch { - c.currentMu.Lock() - defer c.currentMu.Unlock() - - newHeight := c.currentHeight + abi.ChainEpoch(epochs) - - // Create a new tipset at the new height - newTs, err := types.NewTipSet([]*types.BlockHeader{ - { - Height: newHeight, - Miner: c.miner, - // Add necessary parent info - Parents: c.currentTipSet.Key().Cids(), - ParentStateRoot: RandomCID(c.t), - ParentMessageReceipts: RandomCID(c.t), - Messages: RandomCID(c.t), - }, - }) - require.NoError(c.t, err) - - // Update the current height and tipset - c.currentHeight = newHeight - c.currentTipSet = newTs - - // Notify all registered channels - change := []*api.HeadChange{ - { - Type: store.HCApply, - Val: newTs, - }, - } - - for _, ch := range c.notifyChans { - select { - case ch <- change: - // Successfully sent notification - default: - // Channel is full, continue without blocking - } - } - return newHeight -} diff --git a/pkg/pdp/testing/eth_client.go b/pkg/pdp/testing/eth_client.go deleted file mode 100644 index e5504de6..00000000 --- a/pkg/pdp/testing/eth_client.go +++ /dev/null @@ -1,40 +0,0 @@ -package testing - -import ( - "context" - "math/big" - - "go.uber.org/mock/gomock" - - "github.com/storacha/piri/internal/mocks" -) - -// MockEthClient combines all mock interfaces for EthClient -type MockEthClient struct { - *mocks.MockSenderETHClient - *mocks.MockMessageWatcherEthClient - *MockContractBackendWrapper -} - -// MockContractBackendWrapper wraps MockContractBackend but excludes the conflicting method -type MockContractBackendWrapper struct { - *mocks.MockContractBackend -} - -// SuggestGasTipCap delegates to MockSenderETHClient's implementation -// This overrides the method from MockContractBackend to avoid the conflict -func (m *MockEthClient) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { - // Always use the SenderETHClient implementation - return m.MockSenderETHClient.SuggestGasTipCap(ctx) -} - -// NewMockEthClient creates a new mock instance that implements all required interfaces -func NewMockEthClient(ctrl *gomock.Controller) *MockEthClient { - return &MockEthClient{ - MockSenderETHClient: mocks.NewMockSenderETHClient(ctrl), - MockMessageWatcherEthClient: mocks.NewMockMessageWatcherEthClient(ctrl), - MockContractBackendWrapper: &MockContractBackendWrapper{ - MockContractBackend: mocks.NewMockContractBackend(ctrl), - }, - } -} diff --git a/pkg/server/server.go b/pkg/server/server.go index 153e8590..5f5caed6 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -2,114 +2,18 @@ package server import ( "encoding/json" - "errors" "fmt" "net/http" "strings" logging "github.com/ipfs/go-log/v2" - "github.com/labstack/echo/v4" - "github.com/storacha/go-libstoracha/ipnipublisher/store" "github.com/storacha/go-ucanto/principal" - "github.com/storacha/go-ucanto/server" - ucanretrieval "github.com/storacha/go-ucanto/server/retrieval" "github.com/storacha/piri/pkg/build" - "github.com/storacha/piri/pkg/service/blobs" - "github.com/storacha/piri/pkg/service/claims" - "github.com/storacha/piri/pkg/service/publisher" - "github.com/storacha/piri/pkg/service/retrieval" - "github.com/storacha/piri/pkg/service/storage" ) var log = logging.Logger("server") -type serverConfig struct { - ucanSrvOpts []server.Option - ucanRetrievalSrvOpts []ucanretrieval.Option -} - -type Option = func(c *serverConfig) - -func WithUCANServerOptions(options ...server.Option) Option { - return func(c *serverConfig) { - c.ucanSrvOpts = options - } -} - -func WithUCANRetrievalServerOptions(options ...ucanretrieval.Option) Option { - return func(c *serverConfig) { - c.ucanRetrievalSrvOpts = options - } -} - -// ListenAndServe creates a new storage node HTTP server, and starts it up. -func ListenAndServe(addr string, storageSvc storage.Service, retrievalSvc retrieval.Service, options ...Option) error { - srvMux, err := NewServer(storageSvc, retrievalSvc, options...) - if err != nil { - return err - } - srv := &http.Server{ - Addr: addr, - Handler: srvMux, - } - log.Infof("Listening on %s", addr) - err = srv.ListenAndServe() - if err != nil && !errors.Is(err, http.ErrServerClosed) { - return err - } - return nil -} - -// NewServer creates a new storage node server. -func NewServer(storageSvc storage.Service, retrievalSvc retrieval.Service, options ...Option) (*echo.Echo, error) { - cfg := serverConfig{} - for _, opt := range options { - opt(&cfg) - } - - mux := echo.New() - mux.GET("/", echo.WrapHandler(NewHandler(storageSvc.ID()))) - - httpUcanSrv, err := storage.NewServer(storageSvc, cfg.ucanSrvOpts...) - if err != nil { - return nil, fmt.Errorf("creating UCAN server: %w", err) - } - httpUcanSrv.RegisterRoutes(mux) - - httpUcanRetrievalSrv, err := retrieval.NewServer(retrievalSvc, cfg.ucanRetrievalSrvOpts...) - if err != nil { - return nil, fmt.Errorf("creating UCAN retrieval server: %w", err) - } - httpUcanRetrievalSrv.RegisterRoutes(mux) - - httpClaimsSrv, err := claims.NewServer(storageSvc.Claims().Store()) - if err != nil { - return nil, fmt.Errorf("creating claims server: %w", err) - } - httpClaimsSrv.RegisterRoutes(mux) - - httpBlobsSrv, err := blobs.NewServer(storageSvc.Blobs().Presigner(), storageSvc.Blobs().Allocations(), storageSvc.Blobs().Store()) - if err != nil { - return nil, fmt.Errorf("creating blobs server: %w", err) - } - httpBlobsSrv.RegisterRoutes(mux) - - publisherStore := storageSvc.Claims().Publisher().Store() - encodableStore, ok := publisherStore.(store.EncodeableStore) - if !ok { - return nil, errors.New("publisher store does not implement EncodableStore") - } - - httpPublisherSrv, err := publisher.NewServer(encodableStore) - if err != nil { - return nil, fmt.Errorf("creating IPNI publisher server: %w", err) - } - httpPublisherSrv.RegisterRoutes(mux) - - return mux, nil -} - type ServerInfo struct { ID string `json:"id"` Build BuildInfo `json:"build"` diff --git a/pkg/service/claims/options.go b/pkg/service/claims/options.go deleted file mode 100644 index e97e5489..00000000 --- a/pkg/service/claims/options.go +++ /dev/null @@ -1,98 +0,0 @@ -package claims - -import ( - "net/url" - - logging "github.com/ipfs/go-log/v2" - "github.com/multiformats/go-multiaddr" - "github.com/storacha/go-libstoracha/ipnipublisher/publisher" - "github.com/storacha/go-ucanto/client" - "github.com/storacha/go-ucanto/core/delegation" - "github.com/storacha/go-ucanto/transport/http" - "github.com/storacha/go-ucanto/ucan" -) - -type options struct { - asyncPublisher publisher.AsyncPublisher - announceAddr multiaddr.Multiaddr - announceURLs []url.URL - blobAddr multiaddr.Multiaddr - indexingService client.Connection - indexingServiceProofs delegation.Proofs -} - -type Option func(*options) error - -// WithAsyncPublisher configures the async publisher for IPNI advertisements (overrides any publisher specific config) -func WithAsyncPublisher(p publisher.AsyncPublisher) Option { - return func(o *options) error { - o.asyncPublisher = p - return nil - } -} - -// WithPublisherAnnounceAddress sets the address put into announce messages to -// tell indexers where to fetch advertisements from. -func WithPublisherAnnounceAddress(addr multiaddr.Multiaddr) Option { - return func(o *options) error { - o.announceAddr = addr - return nil - } -} - -// WithPublisherBlobAddress sets the address the publisher uses to announce blobs -func WithPublisherBlobAddress(addr multiaddr.Multiaddr) Option { - return func(o *options) error { - o.blobAddr = addr - return nil - } -} - -// WithPublisherDirectAnnounce sets indexer URLs to send direct HTTP -// announcements to. -func WithPublisherDirectAnnounce(announceURLs ...url.URL) Option { - return func(o *options) error { - o.announceURLs = append(o.announceURLs, announceURLs...) - return nil - } -} - -// WithPublisherIndexingService sets the client connection to the indexing UCAN -// service. -func WithPublisherIndexingService(conn client.Connection) Option { - return func(opts *options) error { - opts.indexingService = conn - return nil - } -} - -// WithPublisherIndexingServiceConfig configures UCAN service invocation details -// for communicating with the indexing service. -func WithPublisherIndexingServiceConfig(serviceDID ucan.Principal, serviceURL url.URL) Option { - return func(opts *options) error { - channel := http.NewChannel(&serviceURL) - conn, err := client.NewConnection(serviceDID, channel) - if err != nil { - return err - } - opts.indexingService = conn - return nil - } -} - -// WithPublisherIndexingServiceProof configures proofs for UCAN invocations to -// the indexing service. -func WithPublisherIndexingServiceProof(proof ...delegation.Proof) Option { - return func(opts *options) error { - opts.indexingServiceProofs = proof - return nil - } -} - -// WithLogLevel changes the log level for the claims subsystem. -func WithLogLevel(level string) Option { - return func(c *options) error { - logging.SetLogLevel("claims", level) - return nil - } -} diff --git a/pkg/service/claims/service.go b/pkg/service/claims/service.go index 3b185cf6..622dee71 100644 --- a/pkg/service/claims/service.go +++ b/pkg/service/claims/service.go @@ -1,10 +1,6 @@ package claims import ( - "github.com/multiformats/go-multiaddr" - "github.com/storacha/go-libstoracha/ipnipublisher/store" - "github.com/storacha/go-ucanto/principal" - "github.com/storacha/piri/pkg/service/publisher" "github.com/storacha/piri/pkg/store/claimstore" ) @@ -24,34 +20,7 @@ func (c *ClaimService) Store() claimstore.ClaimStore { var _ Claims = (*ClaimService)(nil) -func New(id principal.Signer, claimStore claimstore.ClaimStore, publisherStore store.PublisherStore, publicAddr multiaddr.Multiaddr, opts ...Option) (*ClaimService, error) { - o := &options{} - for _, opt := range opts { - err := opt(o) - if err != nil { - return nil, err - } - } - - publisher, err := publisher.New( - id, - publisherStore, - publicAddr, - publisher.WithAsyncPublisher(o.asyncPublisher), - publisher.WithDirectAnnounce(o.announceURLs...), - publisher.WithIndexingService(o.indexingService), - publisher.WithIndexingServiceProof(o.indexingServiceProofs...), - publisher.WithAnnounceAddress(o.announceAddr), - publisher.WithBlobAddress(o.blobAddr), - ) - if err != nil { - return nil, err - } - - return &ClaimService{claimStore, publisher}, nil -} - -func NewV2( +func New( claimStore claimstore.ClaimStore, publisher publisher.Publisher, ) *ClaimService { diff --git a/pkg/service/retrieval/server.go b/pkg/service/retrieval/server.go index 0aa3196f..f9d2acdd 100644 --- a/pkg/service/retrieval/server.go +++ b/pkg/service/retrieval/server.go @@ -13,14 +13,6 @@ type Server struct { server server.ServerView[retrieval.Service] } -func NewServer(service Service, options ...retrieval.Option) (*Server, error) { - retrievalSrv, err := NewUCANServer(service, options...) - if err != nil { - return nil, fmt.Errorf("creating UCAN retrieval server: %w", err) - } - return &Server{retrievalSrv}, nil -} - func (srv *Server) RegisterRoutes(e *echo.Echo) { e.GET("/piece/:cid", NewHandler(srv.server)) } diff --git a/pkg/service/retrieval/ucan.go b/pkg/service/retrieval/ucan.go deleted file mode 100644 index 7b2e024e..00000000 --- a/pkg/service/retrieval/ucan.go +++ /dev/null @@ -1,17 +0,0 @@ -package retrieval - -import ( - "github.com/storacha/go-ucanto/server" - "github.com/storacha/go-ucanto/server/retrieval" - "github.com/storacha/piri/pkg/service/retrieval/ucan" -) - -func NewUCANServer(retrievalService Service, options ...retrieval.Option) (server.ServerView[retrieval.Service], error) { - options = append( - options, - ucan.WithBlobRetrieveMethod(retrievalService), - ucan.WithSpaceContentRetrieveMethod(retrievalService), - ) - - return retrieval.NewServer(retrievalService.ID(), options...) -} diff --git a/pkg/service/retrieval/ucan_test.go b/pkg/service/retrieval/ucan_test.go deleted file mode 100644 index ca4df6c1..00000000 --- a/pkg/service/retrieval/ucan_test.go +++ /dev/null @@ -1,150 +0,0 @@ -package retrieval_test - -import ( - "bytes" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "testing" - - "github.com/ipfs/go-cid" - "github.com/storacha/go-libstoracha/capabilities/space/content" - "github.com/storacha/go-libstoracha/testutil" - "github.com/storacha/go-ucanto/client" - retrievalclient "github.com/storacha/go-ucanto/client/retrieval" - "github.com/storacha/go-ucanto/core/delegation" - "github.com/storacha/go-ucanto/core/invocation" - "github.com/storacha/go-ucanto/core/receipt" - "github.com/storacha/go-ucanto/core/result" - ucanhttp "github.com/storacha/go-ucanto/transport/http" - "github.com/storacha/go-ucanto/ucan" - piritutil "github.com/storacha/piri/pkg/internal/testutil" - "github.com/stretchr/testify/require" - - "github.com/storacha/piri/pkg/server" - "github.com/storacha/piri/pkg/service/retrieval" - "github.com/storacha/piri/pkg/service/storage" - "github.com/storacha/piri/pkg/store/allocationstore/allocation" -) - -func TestSpaceContentRetrieve(t *testing.T) { - ctx := t.Context() - uploadServiceConn := testutil.Must(client.NewConnection(testutil.Service.DID(), ucanhttp.NewChannel(testutil.TestURL)))(t) - storageSvc, err := storage.New(uploadServiceConn, storage.WithIdentity(testutil.Alice), storage.WithLogLevel("*", "warn")) - require.NoError(t, err) - err = storageSvc.Startup(ctx) - require.NoError(t, err) - t.Cleanup(func() { - storageSvc.Close(ctx) - }) - - retrievalSvc := retrieval.New(testutil.Alice, storageSvc.Blobs().Store(), storageSvc.Blobs().Allocations()) - - port := piritutil.GetFreePort(t) - srvMux, err := server.NewServer(storageSvc, retrievalSvc) - require.NoError(t, err) - srv := &http.Server{ - Addr: fmt.Sprintf("localhost:%d", port), - Handler: srvMux, - } - go func() { - err = srv.ListenAndServe() - if err != nil && !errors.Is(err, http.ErrServerClosed) { - require.NoError(t, err) - } - }() - t.Cleanup(func() { - srv.Close() - }) - publicURL := testutil.Must(url.Parse(fmt.Sprintf("http://localhost:%d", port)))(t) - - t.Run("space/content/retrieve", func(t *testing.T) { - space := testutil.RandomSigner(t) - randBytes := testutil.RandomBytes(t, 256) - blob := struct { - bytes []byte - cid cid.Cid - }{randBytes, cid.NewCidV1(cid.Raw, testutil.MultihashFromBytes(t, randBytes))} - - storageSvc.Blobs().Allocations().Put(t.Context(), allocation.Allocation{ - Blob: allocation.Blob{ - Digest: blob.cid.Hash(), - Size: uint64(len(blob.bytes)), - }, - Space: space.DID(), - Expires: 0, - Cause: testutil.RandomCID(t), - }) - err := storageSvc.Blobs().Store().Put( - t.Context(), - blob.cid.Hash(), - uint64(len(blob.bytes)), - bytes.NewReader(blob.bytes), - ) - require.NoError(t, err) - - prf := delegation.FromDelegation( - testutil.Must( - delegation.Delegate( - space, - testutil.Bob, - []ucan.Capability[content.RetrieveCaveats]{ - ucan.NewCapability( - content.RetrieveAbility, - space.DID().String(), - content.RetrieveCaveats{ - Blob: content.BlobDigest{Digest: blob.cid.Hash()}, - Range: content.Range{Start: 0, End: uint64(len(blob.bytes) - 1)}, - }, - ), - }, - ), - )(t), - ) - - url := publicURL.JoinPath("piece", blob.cid.String()) - conn, err := retrievalclient.NewConnection(testutil.Alice, url) - require.NoError(t, err) - - inv, err := invocation.Invoke( - testutil.Bob, - testutil.Alice, - content.Retrieve.New( - space.DID().String(), - content.RetrieveCaveats{ - Blob: content.BlobDigest{Digest: blob.cid.Hash()}, - Range: content.Range{Start: 0, End: 1}, - }, - ), - delegation.WithProof(prf), - ) - require.NoError(t, err) - - xres, hres, err := retrievalclient.Execute(t.Context(), inv, conn) - require.NoError(t, err) - - expectStatus := http.StatusPartialContent - expectHeaders := http.Header{ - http.CanonicalHeaderKey("Content-Length"): []string{fmt.Sprintf("%d", 2)}, - http.CanonicalHeaderKey("Content-Range"): []string{fmt.Sprintf("bytes %d-%d/%d", 0, 1, len(blob.bytes))}, - } - expectBody := blob.bytes[0:2] - - require.Equal(t, expectStatus, hres.Status()) - for k, v := range expectHeaders { - require.Equal(t, v, hres.Headers().Values(k)) - } - require.Equal(t, expectBody, testutil.Must(io.ReadAll(hres.Body()))(t)) - - rcptLink, ok := xres.Get(inv.Link()) - require.True(t, ok) - - rcpt, err := receipt.NewAnyReceiptReader().Read(rcptLink, xres.Blocks()) - require.NoError(t, err) - - _, x := result.Unwrap(rcpt.Out()) - require.Nil(t, x) - }) -} diff --git a/pkg/service/storage/options.go b/pkg/service/storage/options.go deleted file mode 100644 index 50f27738..00000000 --- a/pkg/service/storage/options.go +++ /dev/null @@ -1,284 +0,0 @@ -package storage - -import ( - "database/sql" - "net/url" - - "github.com/ipfs/go-datastore" - logging "github.com/ipfs/go-log/v2" - "github.com/multiformats/go-multiaddr" - "github.com/storacha/go-libstoracha/ipnipublisher/publisher" - "github.com/storacha/go-libstoracha/ipnipublisher/store" - "github.com/storacha/go-ucanto/client" - "github.com/storacha/go-ucanto/core/delegation" - "github.com/storacha/go-ucanto/principal" - "github.com/storacha/go-ucanto/transport/http" - "github.com/storacha/go-ucanto/ucan" - "github.com/storacha/go-ucanto/validator" - - "github.com/storacha/piri/pkg/access" - "github.com/storacha/piri/pkg/presigner" - "github.com/storacha/piri/pkg/store/acceptancestore" - "github.com/storacha/piri/pkg/store/allocationstore" - "github.com/storacha/piri/pkg/store/blobstore" - "github.com/storacha/piri/pkg/store/claimstore" - "github.com/storacha/piri/pkg/store/receiptstore" -) - -type config struct { - id principal.Signer - publicURL url.URL - blobsPublicURL url.URL - blobsPresigner presigner.RequestPresigner - blobStore blobstore.Blobstore - blobsAccess access.Access - allocationStore allocationstore.AllocationStore - allocationDatastore datastore.Datastore - acceptanceStore acceptancestore.AcceptanceStore - acceptanceDatastore datastore.Datastore - claimStore claimstore.ClaimStore - claimDatastore datastore.Datastore - asyncPublisher publisher.AsyncPublisher - publisherStore store.PublisherStore - publisherDatastore datastore.Datastore - publisherAnnouceAddr multiaddr.Multiaddr - publisherBlobAddress multiaddr.Multiaddr - receiptStore receiptstore.ReceiptStore - receiptDatastore datastore.Datastore - announceURLs []url.URL - indexingService client.Connection - indexingServiceProofs delegation.Proofs - replicatorDB *sql.DB - claimCtx validator.ClaimContext -} - -type Option func(*config) error - -// WithIdentity configures the storage service identity, used to sign UCAN -// invocations and receipts. -func WithIdentity(signer principal.Signer) Option { - return func(c *config) error { - c.id = signer - return nil - } -} - -// WithPublicURL configures the URL this storage node will be publically -// accessible from. -func WithPublicURL(url url.URL) Option { - return func(c *config) error { - c.publicURL = url - return nil - } -} - -// WithBlobstore configures the blob storage to use. -func WithBlobstore(blobStore blobstore.Blobstore) Option { - return func(c *config) error { - c.blobStore = blobStore - return nil - } -} - -// WithBlobsPublicURL configures the blob storage to use a public URL -func WithBlobsPublicURL(blobStorePublicURL url.URL) Option { - return func(c *config) error { - c.blobsPublicURL = blobStorePublicURL - return nil - } -} - -// WithBlobsAccess configures the access instance for blob storage. -func WithBlobsAccess(access access.Access) Option { - return func(c *config) error { - c.blobsAccess = access - return nil - } -} - -// WithBlobsPresigner configures the blob storage to use a set presigner -func WithBlobsPresigner(blobStorePresigner presigner.RequestPresigner) Option { - return func(c *config) error { - c.blobsPresigner = blobStorePresigner - return nil - } -} - -// WithAllocationStore configures the allocation store directly -func WithAllocationStore(allocationStore allocationstore.AllocationStore) Option { - return func(c *config) error { - c.allocationStore = allocationStore - return nil - } -} - -// WithAllocationDatastore configures the underlying datastore to use for -// storing allocation records. Note: the datastore MUST have efficient support -// for prefix queries. -func WithAllocationDatastore(dstore datastore.Datastore) Option { - return func(c *config) error { - c.allocationDatastore = dstore - return nil - } -} - -// WithAcceptanceStore configures the acceptance store directly -func WithAcceptanceStore(acceptanceStore acceptancestore.AcceptanceStore) Option { - return func(c *config) error { - c.acceptanceStore = acceptanceStore - return nil - } -} - -// WithAcceptanceDatastore configures the underlying datastore to use for -// storing acceptance records. Note: the datastore MUST have efficient support -// for prefix queries. -func WithAcceptanceDatastore(dstore datastore.Datastore) Option { - return func(c *config) error { - c.acceptanceDatastore = dstore - return nil - } -} - -// WithClaimStore configures the store for content claims directly -func WithClaimStore(claimStore claimstore.ClaimStore) Option { - return func(c *config) error { - c.claimStore = claimStore - return nil - } -} - -// WithClaimDatastore configures the underlying datastore to use for storing -// content claims made by this node. -func WithClaimDatastore(dstore datastore.Datastore) Option { - return func(c *config) error { - c.claimDatastore = dstore - return nil - } -} - -// WithReceiptStore configures the store for receipts directly -func WithReceiptStore(receiptStore receiptstore.ReceiptStore) Option { - return func(c *config) error { - c.receiptStore = receiptStore - return nil - } -} - -// WithReceiptDatastore configures the underlying datastore for use storing receipts -// made for this node -func WithReceiptDatastore(dstore datastore.Datastore) Option { - return func(c *config) error { - c.receiptDatastore = dstore - return nil - } -} - -// WithAsyncPublisher configures the async publisher for IPNI advertisements (overrides any publisher specific config) -func WithAsyncPublisher(p publisher.AsyncPublisher) Option { - return func(c *config) error { - c.asyncPublisher = p - return nil - } -} - -// WithPublisherStore configures the store for IPNI advertisements and their -// entries directly. -func WithPublisherStore(publisherStore store.PublisherStore) Option { - return func(c *config) error { - c.publisherStore = publisherStore - return nil - } -} - -// WithPublisherDatastore configures the underlying datastore to use for storing -// IPNI advertisements and their entries. -func WithPublisherDatastore(dstore datastore.Datastore) Option { - return func(c *config) error { - c.publisherDatastore = dstore - return nil - } -} - -// WithPublisherAnnounceAddress sets the address put into announce messages to -// tell indexers where to fetch advertisements from. -func WithPublisherAnnounceAddress(addr multiaddr.Multiaddr) Option { - return func(c *config) error { - c.publisherAnnouceAddr = addr - return nil - } -} - -// WithPublisherBlobAddress sets the multiaddr for blobs used by the publisher -func WithPublisherBlobAddress(addr multiaddr.Multiaddr) Option { - return func(c *config) error { - c.publisherBlobAddress = addr - return nil - } -} - -// WithPublisherDirectAnnounce sets IPNI node URLs to send direct HTTP -// announcements to. -func WithPublisherDirectAnnounce(announceURLs ...url.URL) Option { - return func(c *config) error { - c.announceURLs = append(c.announceURLs, announceURLs...) - return nil - } -} - -// WithPublisherIndexingService sets the client connection to the indexing UCAN -// service. -func WithPublisherIndexingService(conn client.Connection) Option { - return func(c *config) error { - c.indexingService = conn - return nil - } -} - -// WithPublisherIndexingServiceConfig configures UCAN service invocation details -// for communicating with the indexing service. -func WithPublisherIndexingServiceConfig(serviceDID ucan.Principal, serviceURL url.URL) Option { - return func(c *config) error { - channel := http.NewChannel(&serviceURL) - conn, err := client.NewConnection(serviceDID, channel) - if err != nil { - return err - } - c.indexingService = conn - return nil - } -} - -// WithPublisherIndexingServiceProof configures proofs for UCAN invocations to -// the indexing service. -func WithPublisherIndexingServiceProof(proof ...delegation.Proof) Option { - return func(c *config) error { - c.indexingServiceProofs = proof - return nil - } -} - -// WithLogLevel changes the log level of a specific subsystem name=="*" changes -// all subsystems. -func WithLogLevel(name string, level string) Option { - return func(c *config) error { - logging.SetLogLevel(name, level) - return nil - } -} - -func WithReplicatorDB(db *sql.DB) Option { - return func(c *config) error { - c.replicatorDB = db - return nil - } -} - -// WithClaimValidationContext configures the validation context for use when -// validating UCANs. -func WithClaimValidationContext(ctx validator.ClaimContext) Option { - return func(c *config) error { - c.claimCtx = ctx - return nil - } -} diff --git a/pkg/service/storage/server.go b/pkg/service/storage/server.go index 570d90fc..59c99e4d 100644 --- a/pkg/service/storage/server.go +++ b/pkg/service/storage/server.go @@ -6,6 +6,7 @@ import ( "github.com/labstack/echo/v4" "github.com/storacha/go-ucanto/server" ucanhttp "github.com/storacha/go-ucanto/transport/http" + "github.com/storacha/piri/pkg/server/handler" ) @@ -13,15 +14,6 @@ type Server struct { ucanServer server.ServerView[server.Service] } -func NewServer(service Service, options ...server.Option) (*Server, error) { - ucanSrv, err := NewUCANServer(service, options...) - if err != nil { - return nil, fmt.Errorf("creating UCAN server: %w", err) - } - - return &Server{ucanSrv}, nil -} - func (srv *Server) RegisterRoutes(e *echo.Echo) { handler := NewHandler(srv.ucanServer).ToEcho() e.POST("/", handler) diff --git a/pkg/service/storage/service.go b/pkg/service/storage/service.go deleted file mode 100644 index e82aa75a..00000000 --- a/pkg/service/storage/service.go +++ /dev/null @@ -1,315 +0,0 @@ -package storage - -import ( - "context" - "errors" - "fmt" - "io" - "net/url" - "runtime" - - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/sync" - "github.com/ipni/go-libipni/maurl" - "github.com/storacha/go-libstoracha/ipnipublisher/store" - "github.com/storacha/go-libstoracha/metadata" - "github.com/storacha/go-ucanto/client" - "github.com/storacha/go-ucanto/principal" - ed25519 "github.com/storacha/go-ucanto/principal/ed25519/signer" - edverifier "github.com/storacha/go-ucanto/principal/ed25519/verifier" - "github.com/storacha/go-ucanto/validator" - - "github.com/storacha/piri/lib/jobqueue" - "github.com/storacha/piri/lib/jobqueue/serializer" - "github.com/storacha/piri/pkg/database/sqlitedb" - "github.com/storacha/piri/pkg/pdp" - "github.com/storacha/piri/pkg/service/blobs" - "github.com/storacha/piri/pkg/service/claims" - "github.com/storacha/piri/pkg/service/replicator" - replicahandler "github.com/storacha/piri/pkg/service/storage/handlers/replica" - "github.com/storacha/piri/pkg/store/acceptancestore" - "github.com/storacha/piri/pkg/store/blobstore" - "github.com/storacha/piri/pkg/store/delegationstore" - "github.com/storacha/piri/pkg/store/receiptstore" -) - -type StorageService struct { - id principal.Signer - blobs blobs.Blobs - claims claims.Claims - receiptStore receiptstore.ReceiptStore - replicator replicator.Replicator - uploadService client.Connection - claimCtx validator.ClaimContext - startFuncs []func(ctx context.Context) error - closeFuncs []func(ctx context.Context) error - io.Closer -} - -func (s *StorageService) Replicator() replicator.Replicator { - return s.replicator -} - -func (s *StorageService) UploadConnection() client.Connection { - return s.uploadService -} - -func (s *StorageService) Blobs() blobs.Blobs { - return s.blobs -} - -func (s *StorageService) Claims() claims.Claims { - return s.claims -} - -func (s *StorageService) ID() principal.Signer { - return s.id -} - -func (s *StorageService) PDP() pdp.PDP { - // This instance of the storage service does not support PDP - return nil -} - -func (s *StorageService) Receipts() receiptstore.ReceiptStore { - return s.receiptStore -} - -func (s *StorageService) Startup(ctx context.Context) error { - var err error - for _, startFunc := range s.startFuncs { - err = errors.Join(startFunc(ctx)) - } - s.startFuncs = []func(ctx context.Context) error{} - return err -} - -func (s *StorageService) Close(ctx context.Context) error { - var err error - for _, close := range s.closeFuncs { - err = errors.Join(close(ctx)) - } - s.closeFuncs = []func(context.Context) error{} - return err -} - -func (s *StorageService) ClaimValidationContext() validator.ClaimContext { - return s.claimCtx -} - -var _ Service = (*StorageService)(nil) - -func New(uploadServiceConn client.Connection, opts ...Option) (*StorageService, error) { - c := &config{} - for _, opt := range opts { - err := opt(c) - if err != nil { - return nil, err - } - } - - id := c.id - if id == nil { - log.Warn("Generating a server identity as one has not been configured!") - signer, err := ed25519.Generate() - if err != nil { - return nil, err - } - id = signer - } - log.Infof("Server ID: %s", id.DID()) - - var closeFuncs []func(context.Context) error - var startFuncs []func(ctx context.Context) error - - blobOpts := []blobs.Option{} - - if c.allocationStore == nil { - allocDs := c.allocationDatastore - if allocDs == nil { - allocDs = datastore.NewMapDatastore() - log.Warn("Allocation datastore not configured, using in-memory datastore") - } - closeFuncs = append(closeFuncs, func(context.Context) error { return allocDs.Close() }) - blobOpts = append(blobOpts, blobs.WithDSAllocationStore(allocDs)) - } else { - blobOpts = append(blobOpts, blobs.WithAllocationStore(c.allocationStore)) - } - - if c.acceptanceStore == nil { - acceptDs := c.acceptanceDatastore - if acceptDs == nil { - acceptDs = datastore.NewMapDatastore() - log.Warn("Acceptance datastore not configured, using in-memory datastore") - } - closeFuncs = append(closeFuncs, func(context.Context) error { return acceptDs.Close() }) - acceptanceStore := acceptancestore.NewDatastoreStore(acceptDs) - blobOpts = append(blobOpts, blobs.WithAcceptanceStore(acceptanceStore)) - } else { - blobOpts = append(blobOpts, blobs.WithAcceptanceStore(c.acceptanceStore)) - } - - claimStore := c.claimStore - if claimStore == nil { - claimDs := c.claimDatastore - if claimDs == nil { - claimDs = datastore.NewMapDatastore() - log.Warn("Claim datastore not configured, using in-memory datastore") - } - closeFuncs = append(closeFuncs, func(context.Context) error { return claimDs.Close() }) - claimStore = delegationstore.NewDatastoreStore(claimDs) - } - publisherStore := c.publisherStore - if publisherStore == nil { - publisherDs := c.publisherDatastore - if publisherDs == nil { - publisherDs = datastore.NewMapDatastore() - log.Warn("Publisher datastore not configured, using in-memory datastore") - } - closeFuncs = append(closeFuncs, func(context.Context) error { return publisherDs.Close() }) - publisherStore = store.FromDatastore(publisherDs, store.WithMetadataContext(metadata.MetadataContext)) - } - pubURL := c.publicURL - if pubURL == (url.URL{}) { - u, _ := url.Parse("http://localhost:3000") - log.Warnf("Public URL not configured, using default: %s", u) - pubURL = *u - } - - receiptStore := c.receiptStore - if receiptStore == nil { - receiptDS := c.receiptDatastore - if receiptDS == nil { - receiptDS = datastore.NewMapDatastore() - log.Warn("Receipt datastore not configured, using in-memory datastore") - } - closeFuncs = append(closeFuncs, func(context.Context) error { return receiptDS.Close() }) - receiptStore = receiptstore.NewDatastoreStore(receiptDS) - } - - blobStore := c.blobStore - if blobStore == nil { - blobStore = blobstore.NewDatastoreStore(sync.MutexWrap(datastore.NewMapDatastore())) - log.Warn("Blob store not configured, using in-memory store") - } - - blobOpts = append(blobOpts, blobs.WithBlobstore(blobStore)) - if c.blobsAccess != nil { - blobOpts = append(blobOpts, blobs.WithAccess(c.blobsAccess)) - } else if c.blobsPublicURL != (url.URL{}) { - blobOpts = append(blobOpts, blobs.WithPublicURLAccess(c.blobsPublicURL)) - } else { - blobOpts = append(blobOpts, blobs.WithPublicURLAccess(pubURL)) - } - - if c.blobsPresigner != nil { - blobOpts = append(blobOpts, blobs.WithPresigner(c.blobsPresigner)) - } else if c.blobsPublicURL != (url.URL{}) { - blobOpts = append(blobOpts, blobs.WithPublicURLPresigner(id, c.blobsPublicURL)) - } else { - blobOpts = append(blobOpts, blobs.WithPublicURLPresigner(id, pubURL)) - } - - if uploadServiceConn == nil { - return nil, errors.New("upload service connection cannot be nil") - } - - blobs, err := blobs.New(blobOpts...) - if err != nil { - return nil, fmt.Errorf("creating blob service: %w", err) - } - - peerAddr, err := maurl.FromURL(&pubURL) - if err != nil { - return nil, fmt.Errorf("parsing publisher url as multiaddr: %w", err) - } - - claims, err := claims.New( - id, - claimStore, - publisherStore, - peerAddr, - claims.WithAsyncPublisher(c.asyncPublisher), - claims.WithPublisherDirectAnnounce(c.announceURLs...), - claims.WithPublisherAnnounceAddress(c.publisherAnnouceAddr), - claims.WithPublisherBlobAddress(c.publisherBlobAddress), - claims.WithPublisherIndexingService(c.indexingService), - claims.WithPublisherIndexingServiceProof(c.indexingServiceProofs...), - ) - if err != nil { - return nil, fmt.Errorf("creating claim service: %w", err) - } - - if c.replicatorDB == nil { - c.replicatorDB, err = sqlitedb.NewMemory() - if err != nil { - return nil, fmt.Errorf("creating in-memory replicator db: %w", err) - } - } - - // Create replication queue - replicationQueue, err := jobqueue.New[*replicahandler.TransferRequest]( - "replication", - c.replicatorDB, - &serializer.JSON[*replicahandler.TransferRequest]{}, - jobqueue.WithLogger(log.With("queue", "replication")), - jobqueue.WithMaxRetries(10), - jobqueue.WithMaxWorkers(uint(runtime.NumCPU())), - ) - if err != nil { - return nil, fmt.Errorf("creating replication queue: %w", err) - } - - // replicator does not require a PDP service, so we pass nil. - repl, err := replicator.New(id, nil, blobs, claims, receiptStore, uploadServiceConn, replicationQueue) - if err != nil { - return nil, fmt.Errorf("creating replicator service: %w", err) - } - - // Register transfer task - if err := repl.RegisterTransferTask(replicationQueue); err != nil { - return nil, fmt.Errorf("registering replicator transfer task: %w", err) - } - - // Queue lifecycle management - var queueCtx context.Context - var queueCancel context.CancelFunc - startFuncs = append(startFuncs, func(ctx context.Context) error { - queueCtx, queueCancel = context.WithCancel(context.Background()) - return replicationQueue.Start(queueCtx) - }) - closeFuncs = append(closeFuncs, func(ctx context.Context) error { - if queueCancel != nil { - queueCancel() - } - return replicationQueue.Stop(ctx) - }) - - claimCtx := c.claimCtx - if claimCtx == nil { - log.Warn("Claim validation context not configured - this may cause runtime issues") - claimCtx = validator.NewClaimContext( - id.Verifier(), - validator.IsSelfIssued, - func(context.Context, validator.Authorization[any]) validator.Revoked { - return nil - }, - validator.ProofUnavailable, - edverifier.Parse, - validator.FailDIDKeyResolution, - validator.NotExpiredNotTooEarly, - ) - } - - return &StorageService{ - id: c.id, - blobs: blobs, - claims: claims, - closeFuncs: closeFuncs, - startFuncs: startFuncs, - receiptStore: receiptStore, - replicator: repl, - uploadService: uploadServiceConn, - claimCtx: claimCtx, - }, nil -} diff --git a/pkg/service/storage/ucan.go b/pkg/service/storage/ucan.go deleted file mode 100644 index 1007e493..00000000 --- a/pkg/service/storage/ucan.go +++ /dev/null @@ -1,23 +0,0 @@ -package storage - -import ( - logging "github.com/ipfs/go-log/v2" - "github.com/storacha/go-ucanto/server" - - "github.com/storacha/piri/pkg/service/storage/ucan" -) - -var log = logging.Logger("storage") - -func NewUCANServer(storageService Service, options ...server.Option) (server.ServerView[server.Service], error) { - options = append( - options, - ucan.WithAccessGrantMethod(storageService), - ucan.WithBlobAllocateMethod(storageService), - ucan.WithBlobAcceptMethod(storageService), - ucan.WithPDPInfoMethod(storageService), - ucan.WithReplicaAllocateMethod(storageService), - ) - - return server.NewServer(storageService.ID(), options...) -} diff --git a/pkg/service/storage/ucan_test.go b/pkg/service/storage/ucan_test.go deleted file mode 100644 index 5991dd4a..00000000 --- a/pkg/service/storage/ucan_test.go +++ /dev/null @@ -1,981 +0,0 @@ -package storage - -import ( - "bytes" - "context" - "fmt" - "io" - "math/rand/v2" - "net/http" - "net/url" - "testing" - "time" - - "github.com/ipfs/go-cid" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/multiformats/go-multihash" - "github.com/storacha/go-libstoracha/capabilities/access" - "github.com/storacha/go-libstoracha/capabilities/assert" - "github.com/storacha/go-libstoracha/capabilities/blob" - "github.com/storacha/go-libstoracha/capabilities/blob/replica" - blob2 "github.com/storacha/go-libstoracha/capabilities/space/blob" - "github.com/storacha/go-libstoracha/capabilities/space/content" - "github.com/storacha/go-libstoracha/capabilities/types" - ucancap "github.com/storacha/go-libstoracha/capabilities/ucan" - "github.com/storacha/go-libstoracha/failure" - "github.com/storacha/go-ucanto/client" - "github.com/storacha/go-ucanto/core/car" - "github.com/storacha/go-ucanto/core/dag/blockstore" - "github.com/storacha/go-ucanto/core/delegation" - "github.com/storacha/go-ucanto/core/invocation" - "github.com/storacha/go-ucanto/core/ipld" - "github.com/storacha/go-ucanto/core/message" - "github.com/storacha/go-ucanto/core/receipt" - "github.com/storacha/go-ucanto/core/receipt/ran" - "github.com/storacha/go-ucanto/core/result" - ufailure "github.com/storacha/go-ucanto/core/result/failure" - "github.com/storacha/go-ucanto/core/result/ok" - "github.com/storacha/go-ucanto/did" - ucan_car "github.com/storacha/go-ucanto/transport/car" - "github.com/storacha/go-ucanto/transport/headercar" - ucan_http "github.com/storacha/go-ucanto/transport/http" - "github.com/storacha/go-ucanto/ucan" - testutil2 "github.com/storacha/piri/pkg/internal/testutil" - "github.com/stretchr/testify/require" - - "github.com/storacha/go-libstoracha/testutil" - "github.com/storacha/piri/pkg/store/allocationstore/allocation" -) - -func TestServer(t *testing.T) { - ctx := t.Context() - uploadServiceConn := testutil.Must(client.NewConnection(testutil.Service.DID(), ucan_http.NewChannel(testutil.TestURL)))(t) - svc, err := New(uploadServiceConn, WithIdentity(testutil.Alice), WithLogLevel("*", "warn")) - require.NoError(t, err) - err = svc.Startup(ctx) - require.NoError(t, err) - t.Cleanup(func() { - svc.Close(ctx) - }) - - srv, err := NewUCANServer(svc) - require.NoError(t, err) - - conn := testutil.Must(client.NewConnection(testutil.Service, srv))(t) - - prf := delegation.FromDelegation( - testutil.Must( - delegation.Delegate( - testutil.Alice, - testutil.Service, - []ucan.Capability[ucan.CaveatBuilder]{ - ucan.NewCapability( - blob.AllocateAbility, - testutil.Alice.DID().String(), - ucan.CaveatBuilder(ok.Unit{}), - ), - ucan.NewCapability( - blob.AcceptAbility, - testutil.Alice.DID().String(), - ucan.CaveatBuilder(ok.Unit{}), - ), - }, - ), - )(t), - ) - - t.Run("blob/allocate", func(t *testing.T) { - space := testutil.RandomDID(t) - digest := testutil.RandomMultihash(t) - size := uint64(rand.IntN(32) + 1) - cause := testutil.RandomCID(t) - - nb := blob.AllocateCaveats{ - Space: space, - Blob: types.Blob{ - Digest: digest, - Size: size, - }, - Cause: cause, - } - cap := blob.Allocate.New(testutil.Alice.DID().String(), nb) - inv, err := invocation.Invoke(testutil.Service, testutil.Alice, cap, delegation.WithProof(prf)) - require.NoError(t, err) - - resp, err := client.Execute(ctx, []invocation.Invocation{inv}, conn) - require.NoError(t, err) - - // get the receipt link for the invocation from the response - rcptlnk, ok := resp.Get(inv.Link()) - require.True(t, ok, "missing receipt for invocation: %s", inv.Link()) - - reader := testutil.Must(receipt.NewReceiptReaderFromTypes[blob.AllocateOk, failure.FailureModel](blob.AllocateOkType(), failure.FailureType(), types.Converters...))(t) - rcpt := testutil.Must(reader.Read(rcptlnk, resp.Blocks()))(t) - - result.MatchResultR0(rcpt.Out(), func(ok blob.AllocateOk) { - fmt.Printf("%+v\n", ok) - require.Equal(t, size, uint64(ok.Size)) - - alloc, err := svc.Blobs().Allocations().Get(t.Context(), digest, space) - require.NoError(t, err) - - require.Equal(t, digest, alloc.Blob.Digest) - require.Equal(t, size, alloc.Blob.Size) - require.Equal(t, space, alloc.Space) - require.Equal(t, inv.Link(), alloc.Cause) - }, func(f failure.FailureModel) { - fmt.Println(f.Message) - fmt.Println(*f.Stack) - require.Nil(t, f) - }) - }) - - t.Run("repeat blob/allocate for same blob", func(t *testing.T) { - space := testutil.RandomDID(t) - size := uint64(rand.IntN(32) + 1) - data := testutil.RandomBytes(t, int(size)) - digest := testutil.Must(multihash.Sum(data, multihash.SHA2_256, -1))(t) - cause := testutil.RandomCID(t) - - nb := blob.AllocateCaveats{ - Space: space, - Blob: types.Blob{ - Digest: digest, - Size: size, - }, - Cause: cause, - } - cap := blob.Allocate.New(testutil.Alice.DID().String(), nb) - - invokeBlobAllocate := func() result.Result[blob.AllocateOk, failure.FailureModel] { - inv, err := invocation.Invoke(testutil.Service, testutil.Alice, cap, delegation.WithProof(prf)) - require.NoError(t, err) - - resp, err := client.Execute(ctx, []invocation.Invocation{inv}, conn) - require.NoError(t, err) - - rcptlnk, ok := resp.Get(inv.Link()) - require.True(t, ok, "missing receipt for invocation: %s", inv.Link()) - - reader := testutil.Must(receipt.NewReceiptReaderFromTypes[blob.AllocateOk, failure.FailureModel](blob.AllocateOkType(), failure.FailureType(), types.Converters...))(t) - rcpt := testutil.Must(reader.Read(rcptlnk, resp.Blocks()))(t) - return rcpt.Out() - } - - result.MatchResultR0(invokeBlobAllocate(), func(ok blob.AllocateOk) { - fmt.Printf("%+v\n", ok) - require.Equal(t, size, uint64(ok.Size)) - require.NotNil(t, ok.Address) - }, func(f failure.FailureModel) { - fmt.Println(f.Message) - fmt.Println(*f.Stack) - require.Nil(t, f) - }) - - // now again without upload - result.MatchResultR0(invokeBlobAllocate(), func(ok blob.AllocateOk) { - fmt.Printf("%+v\n", ok) - require.Equal(t, uint64(0), ok.Size) - require.NotNil(t, ok.Address) - }, func(f failure.FailureModel) { - fmt.Println(f.Message) - fmt.Println(*f.Stack) - require.Nil(t, f) - }) - - // simulate a blob upload - err = svc.Blobs().Store().Put(t.Context(), digest, size, bytes.NewReader(data)) - require.NoError(t, err) - - // now again after upload - result.MatchResultR0(invokeBlobAllocate(), func(ok blob.AllocateOk) { - fmt.Printf("%+v\n", ok) - require.Equal(t, uint64(0), ok.Size) - require.Nil(t, ok.Address) - }, func(f failure.FailureModel) { - fmt.Println(f.Message) - fmt.Println(*f.Stack) - require.Nil(t, f) - }) - }) - - t.Run("repeat blob/allocate for same blob in different space", func(t *testing.T) { - space0 := testutil.RandomDID(t) - space1 := testutil.RandomDID(t) - size := uint64(rand.IntN(32) + 1) - data := testutil.RandomBytes(t, int(size)) - digest := testutil.Must(multihash.Sum(data, multihash.SHA2_256, -1))(t) - cause := testutil.RandomCID(t) - - invokeBlobAllocate := func(space did.DID) result.Result[blob.AllocateOk, failure.FailureModel] { - nb := blob.AllocateCaveats{ - Space: space, - Blob: types.Blob{ - Digest: digest, - Size: size, - }, - Cause: cause, - } - cap := blob.Allocate.New(testutil.Alice.DID().String(), nb) - - inv, err := invocation.Invoke(testutil.Service, testutil.Alice, cap, delegation.WithProof(prf)) - require.NoError(t, err) - - resp, err := client.Execute(ctx, []invocation.Invocation{inv}, conn) - require.NoError(t, err) - - rcptlnk, ok := resp.Get(inv.Link()) - require.True(t, ok, "missing receipt for invocation: %s", inv.Link()) - - reader := testutil.Must(receipt.NewReceiptReaderFromTypes[blob.AllocateOk, failure.FailureModel](blob.AllocateOkType(), failure.FailureType(), types.Converters...))(t) - rcpt := testutil.Must(reader.Read(rcptlnk, resp.Blocks()))(t) - return rcpt.Out() - } - - result.MatchResultR0(invokeBlobAllocate(space0), func(ok blob.AllocateOk) { - fmt.Printf("%+v\n", ok) - require.Equal(t, size, uint64(ok.Size)) - require.NotNil(t, ok.Address) - }, func(f failure.FailureModel) { - fmt.Println(f.Message) - fmt.Println(*f.Stack) - require.Nil(t, f) - }) - - // simulate a blob upload - err = svc.Blobs().Store().Put(t.Context(), digest, size, bytes.NewReader(data)) - require.NoError(t, err) - - // now again after upload, but in different space - result.MatchResultR0(invokeBlobAllocate(space1), func(ok blob.AllocateOk) { - fmt.Printf("%+v\n", ok) - require.Equal(t, size, uint64(ok.Size)) - require.Nil(t, ok.Address) - }, func(f failure.FailureModel) { - fmt.Println(f.Message) - fmt.Println(*f.Stack) - require.Nil(t, f) - }) - }) - - t.Run("blob/accept", func(t *testing.T) { - space := testutil.RandomDID(t) - size := uint64(rand.IntN(32) + 1) - data := testutil.RandomBytes(t, int(size)) - digest := testutil.Must(multihash.Sum(data, multihash.SHA2_256, -1))(t) - cause := testutil.RandomCID(t) - - allocNb := blob.AllocateCaveats{ - Space: space, - Blob: types.Blob{ - Digest: digest, - Size: size, - }, - Cause: cause, - } - allocCap := blob.Allocate.New(testutil.Alice.DID().String(), allocNb) - allocInv, err := invocation.Invoke(testutil.Service, testutil.Alice, allocCap, delegation.WithProof(prf)) - require.NoError(t, err) - - _, err = client.Execute(ctx, []invocation.Invocation{allocInv}, conn) - require.NoError(t, err) - - // simulate a blob upload - err = svc.Blobs().Store().Put(t.Context(), digest, size, bytes.NewReader(data)) - require.NoError(t, err) - // get the expected download URL - loc, err := svc.Blobs().Access().GetDownloadURL(digest) - require.NoError(t, err) - - // eventually service will invoke blob/accept - acceptNb := blob.AcceptCaveats{ - Space: space, - Blob: types.Blob{ - Digest: digest, - Size: size, - }, - Put: blob.Promise{ - UcanAwait: blob.Await{ - Selector: ".out.ok", - Link: testutil.RandomCID(t), - }, - }, - } - // fmt.Println(printer.Sprint(testutil.Must(acceptNb.ToIPLD())(t))) - acceptCap := blob.Accept.New(testutil.Alice.DID().String(), acceptNb) - acceptInv, err := invocation.Invoke(testutil.Service, testutil.Alice, acceptCap, delegation.WithProof(prf)) - require.NoError(t, err) - - resp, err := client.Execute(ctx, []invocation.Invocation{acceptInv}, conn) - require.NoError(t, err) - - // get the receipt link for the invocation from the response - rcptlnk, ok := resp.Get(acceptInv.Link()) - require.True(t, ok, "missing receipt for invocation: %s", acceptInv.Link()) - - reader := testutil.Must(receipt.NewReceiptReaderFromTypes[blob.AcceptOk, failure.FailureModel](blob.AcceptOkType(), failure.FailureType(), types.Converters...))(t) - rcpt := testutil.Must(reader.Read(rcptlnk, resp.Blocks()))(t) - - result.MatchResultR0(rcpt.Out(), func(ok blob.AcceptOk) { - fmt.Printf("%+v\n", ok) - - claim, err := svc.Claims().Store().Get(t.Context(), ok.Site) - require.NoError(t, err) - - require.Equal(t, testutil.Alice.DID(), claim.Issuer()) - require.Equal(t, space, claim.Audience().DID()) - require.Equal(t, assert.LocationAbility, claim.Capabilities()[0].Can()) - require.Equal(t, testutil.Alice.DID().String(), claim.Capabilities()[0].With()) - - nb, err := assert.LocationCaveatsReader.Read(claim.Capabilities()[0].Nb()) - require.NoError(t, err) - - require.Equal(t, space, nb.Space) - require.Equal(t, digest, nb.Content.Hash()) - require.Equal(t, loc.String(), nb.Location[0].String()) - - // TODO: assert IPNI advert published - }, func(f failure.FailureModel) { - fmt.Println(f.Message) - fmt.Println(*f.Stack) - require.Nil(t, f) - }) - - require.NotEmpty(t, rcpt.Fx().Fork()) - effect := rcpt.Fx().Fork()[0] - claim, ok := effect.Invocation() - require.True(t, ok) - require.Equal(t, assert.LocationAbility, claim.Capabilities()[0].Can()) - }) -} - -// TestReplicaAllocateTransfer validates the full replica allocation flow in the UCAN server, -// ensuring that invocations are correctly constructed and executed, and that the simulated endpoints -// interact as expected. A lightweight HTTP server (on port picked by OS) is used to simulate external endpoints: -// - "/get": Represents the source node that returns the original blob data. -// - "/put": Emulates the replica node that accepts and stores the blob. -// - "/upload-service": Acts as the upload service by decoding a CAR payload and triggering a transfer receipt. -// -// This test covers three scenarios: -// 1. **NoExistingAllocationNoData:** No previous allocation or stored data exists, so the full blob is transferred. -// 2. **ExistingAllocationNoData:** An allocation record is present (indicating reserved space) but the blob data is not yet stored, -// resulting in no additional data, but involving a transfer -// 3. **ExistingAllocationAndData:** Both an allocation record and the blob data are already present; although a transfer receipt is still produced, -// no redundant data transfer should occur. -func TestReplicaAllocateTransfer(t *testing.T) { - testCases := []struct { - name string - hasExistingAllocation bool - hasExistingData bool - expectedTransferSize uint64 - }{ - { - name: "NoExistingAllocationNoData", - hasExistingAllocation: false, - hasExistingData: false, - }, - { - name: "ExistingAllocationNoData", - hasExistingAllocation: true, - hasExistingData: false, - }, - { - name: "ExistingAllocationAndData", - hasExistingAllocation: true, - hasExistingData: true, - }, - } - - for _, tc := range testCases { - tc := tc // capture range variable - t.Run(tc.name, func(t *testing.T) { - // we expect each test to run in 10 seconds or less. - ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) - - // Common setup: random DID, random data, etc. - expectedSpace := testutil.RandomDID(t) - expectedSize := uint64(rand.IntN(32) + 1) - expectedData := testutil.RandomBytes(t, int(expectedSize)) - expectedDigest := testutil.Must( - multihash.Sum(expectedData, multihash.SHA2_256, -1), - )(t) - replicas := uint(1) - port := testutil2.GetFreePort(t) - serverAddr := fmt.Sprintf(":%d", port) - sourcePath, sinkPath, uploadServicePath := "get", "put", "upload-service" - - // Spin up storage service, using injected values for testing. - locationURL, uploadServiceURL, fakeBlobPresigner := setupURLs(t, serverAddr, sourcePath, sinkPath, uploadServicePath) - svc := setupService(t, ctx, fakeBlobPresigner, uploadServiceURL) - fakeServer, transferOkChan := startTestHTTPServer( - ctx, t, expectedDigest, expectedData, svc, - serverAddr, sourcePath, sinkPath, uploadServicePath, - ) - t.Cleanup(func() { - fakeServer.Close() - svc.Close(ctx) - cancel() - }) - - // Build UCAN server & connection - srv, err := NewUCANServer(svc) - require.NoError(t, err) - conn := testutil.Must(client.NewConnection(testutil.Service, srv))(t) - - // Build UCAN delegation + location claim + replicate invocation - // required ability's for blob replicate - prf := buildDelegationProof(t) - // location claim and blob replicate invocation, simulating an upload-service - lcd, expectedLocationCaveats := buildLocationClaim(t, prf, expectedSpace, expectedDigest, locationURL, expectedSize) - bri, expectedReplicaCaveats := buildReplicateInvocation( - t, lcd, expectedDigest, expectedSize, replicas, - ) - - // Condition: If existing allocation, store an existing allocation - // coverage when an allocation has been made but not transferred. - if tc.hasExistingAllocation { - require.NoError(t, svc.Blobs().Allocations().Put(ctx, allocation.Allocation{ - Space: expectedSpace, - Blob: allocation.Blob{ - Digest: expectedDigest, - Size: expectedSize, - }, - Expires: uint64(time.Now().Add(time.Hour).UTC().Unix()), - Cause: bri.Link(), - })) - } - - // Condition: If existing data, store it in the blob store - // covers when an allocation and replica already exist, meaning no transfer required. - // though we still expect a transfer receipt. - if tc.hasExistingData { - require.NoError(t, svc.blobs.Store().Put( - ctx, expectedDigest, expectedSize, bytes.NewReader(expectedData), - )) - } - - // Build + execute the actual replica.Allocate invocation. - // simulating an upload service sending the invocation to the storage node. - rbi, expectedAllocateCaveats := buildAllocateInvocation( - t, bri, lcd, expectedSpace, expectedDigest, expectedSize, - ) - res, err := client.Execute(ctx, []invocation.Invocation{rbi}, conn) - require.NoError(t, err) - - // The final assertion on the returned allocation size. - // With an existing allocation or existing data, the new allocated - // size is 0, otherwise it’s expectedSize. - var wantSize uint64 - if !tc.hasExistingAllocation && !tc.hasExistingData { - wantSize = expectedSize - } - // read the receipt for the blob allocate, asserting its size is expected value. - alloc := mustReadAllocationReceipt(t, rbi, res) - require.EqualValues(t, wantSize, alloc.Size) - - // Assert that the Site promise field exists and has the correct structure - require.NotNil(t, alloc.Site) - require.Equal(t, replica.AllocateSiteSelector, alloc.Site.UcanAwait.Selector) - - // "Wait" for the transfer invocation to produce a receipt - // simulating the upload-service getting a receipt from this storage node. - ucanConcludeMsg := mustWaitForTransferMsg(t, ctx, transferOkChan) - // expect one invocation and 0 receipts - require.Len(t, ucanConcludeMsg.Invocations(), 1) - // receipt is attached to the invocation, not a reciept in the message - require.Len(t, ucanConcludeMsg.Receipts(), 0) - - // Full read + assertion on the transfer invocation and its ucan chain - mustAssertTransferInvocation( - t, - ucanConcludeMsg, - expectedDigest, - wantSize, - expectedSpace, - expectedLocationCaveats, - expectedAllocateCaveats, - expectedReplicaCaveats, - ) - - }) - } -} - -// Sets up the pre-signed URLs + returns them for use in testing -func setupURLs( - t *testing.T, - serverAddr string, - sourcePath, sinkPath, uploadServicePath string, -) (*url.URL, *url.URL, *FakePresigned) { - makeURL := func(path string) *url.URL { - return testutil.Must( - url.Parse(fmt.Sprintf("http://127.0.0.1%s/%s", serverAddr, path)), - )(t) - } - locationURL := makeURL(sourcePath) - uploadServiceURL := makeURL(uploadServicePath) - presignedURL := makeURL(sinkPath) - fakeBlobPresigner := &FakePresigned{uploadURL: *presignedURL} - return locationURL, uploadServiceURL, fakeBlobPresigner -} - -// Creates + starts your main service -func setupService( - t *testing.T, - ctx context.Context, - fakeBlobPresigner *FakePresigned, - uploadServiceURL *url.URL, -) *StorageService { - uploadServiceConn, err := client.NewConnection(testutil.Service.DID(), ucan_http.NewChannel(uploadServiceURL)) - require.NoError(t, err) - - svc, err := New(uploadServiceConn, - WithIdentity(testutil.Alice), - WithLogLevel("*", "warn"), - WithBlobsPresigner(fakeBlobPresigner), - ) - require.NoError(t, err) - require.NoError(t, svc.Startup(ctx)) - return svc -} - -// Builds the UCAN delegation proof needed for replicate + allocate -func buildDelegationProof(t *testing.T) delegation.Delegation { - caps := []ucan.Capability[ucan.CaveatBuilder]{ - ucan.NewCapability(replica.AllocateAbility, testutil.Alice.DID().String(), ucan.CaveatBuilder(ok.Unit{})), - ucan.NewCapability(blob.AllocateAbility, testutil.Alice.DID().String(), ucan.CaveatBuilder(ok.Unit{})), - ucan.NewCapability(blob.AcceptAbility, testutil.Alice.DID().String(), ucan.CaveatBuilder(ok.Unit{})), - } - d := testutil.Must( - delegation.Delegate(testutil.Alice, testutil.Service, caps), - )(t) - return d -} - -// Builds the location claim -func buildLocationClaim( - t *testing.T, - prf delegation.Delegation, - space did.DID, - digest multihash.Multihash, - locationURL *url.URL, - size uint64, -) (delegation.Delegation, assert.LocationCaveats) { - locCav := assert.LocationCaveats{ - Space: space, - Content: types.FromHash(digest), - Location: []url.URL{*locationURL}, - Range: &assert.Range{Offset: 1, Length: &size}, - } - lcd, err := assert.Location.Delegate( - testutil.Alice, - testutil.Alice.DID(), - testutil.Alice.DID().String(), - locCav, - delegation.WithProof(delegation.FromDelegation(prf)), - ) - require.NoError(t, err) - return lcd, locCav -} - -// Builds the replicate invocation + attaches location claim -func buildReplicateInvocation( - t *testing.T, - lcd delegation.Delegation, - digest multihash.Multihash, - size uint64, - replicas uint, -) (invocation.Invocation, blob2.ReplicateCaveats) { - expectedReplicaCaveats := blob2.ReplicateCaveats{ - Blob: types.Blob{ - Digest: digest, - Size: size, - }, - Replicas: replicas, - Site: lcd.Root().Link(), - } - bri, err := blob2.Replicate.Invoke( - testutil.Alice, - testutil.Alice.DID(), - testutil.Alice.DID().String(), - expectedReplicaCaveats, - ) - require.NoError(t, err) - - // attach location claim blocks - for block, err := range lcd.Blocks() { - require.NoError(t, err) - require.NoError(t, bri.Attach(block)) - } - return bri, expectedReplicaCaveats -} - -// Builds the replica allocate invocation + attaches replicate blocks -func buildAllocateInvocation( - t *testing.T, - bri invocation.Invocation, - lcd delegation.Delegation, - space did.DID, - digest multihash.Multihash, - size uint64, -) (invocation.Invocation, replica.AllocateCaveats) { - expectedAllocateCaveats := replica.AllocateCaveats{ - Space: space, - Blob: types.Blob{Digest: digest, Size: size}, - Site: lcd.Root().Link(), - Cause: bri.Root().Link(), - } - rbi, err := replica.Allocate.Invoke( - testutil.Alice, - testutil.Alice.DID(), - testutil.Alice.DID().String(), - expectedAllocateCaveats, - ) - require.NoError(t, err) - - // attach replicate invocation blocks - for block, err := range bri.Blocks() { - require.NoError(t, err) - require.NoError(t, rbi.Attach(block)) - } - return rbi, expectedAllocateCaveats -} - -// Unwrap and read the receipt that returns the replica.AllocateOk -func mustReadAllocationReceipt( - t *testing.T, - rbi invocation.Invocation, - res client.ExecutionResponse, -) replica.AllocateOk { - reader, err := receipt.NewReceiptReaderFromTypes[replica.AllocateOk, failure.FailureModel]( - replica.AllocateOkType(), failure.FailureType(), types.Converters..., - ) - require.NoError(t, err) - - rcptLink, ok := res.Get(rbi.Link()) - require.True(t, ok) - - rcpt, err := reader.Read(rcptLink, res.Blocks()) - require.NoError(t, err) - - alloc, err := result.Unwrap(result.MapError(rcpt.Out(), failure.FromFailureModel)) - require.NoError(t, err) - return alloc -} - -// Wait for the transfer message from the test HTTP server -func mustWaitForTransferMsg( - t *testing.T, - ctx context.Context, - ch <-chan message.AgentMessage, -) message.AgentMessage { - select { - case <-ctx.Done(): - t.Fatal("test did not produce transfer receipt in time: ", ctx.Err()) - return nil - case ucanConcludeMsg := <-ch: - require.NotNil(t, ucanConcludeMsg) - return ucanConcludeMsg - } -} - -func MustAssertTransferInvocationUcanConcludeReceipt( - t *testing.T, - ucanConcludeMsg message.AgentMessage, - expectedDigest multihash.Multihash, - expectedSize uint64, - expectedSpace did.DID, - expectedLocationCav assert.LocationCaveats, - expectedAllocateCav replica.AllocateCaveats, - expectedReplicaCav blob2.ReplicateCaveats, -) { - // sanity check - require.NotNil(t, ucanConcludeMsg) - - concludeInvocationCid := testutil.Must( - cid.Parse(ucanConcludeMsg.Invocations()[0].String()), - )(t) - reader := testutil.Must( - blockstore.NewBlockReader(blockstore.WithBlocksIterator(ucanConcludeMsg.Blocks())), - )(t) - - concludeCav := mustGetInvocationCaveats[ucancap.ConcludeCaveats]( - t, reader, cidlink.Link{Cid: concludeInvocationCid}, - ucancap.ConcludeCaveatsReader.Read, - ) - someotherreader, err := receipt.NewReceiptReaderFromTypes[replica.TransferOk, failure.FailureModel](replica.TransferOkType(), failure.FailureType(), types.Converters...) - require.NoError(t, err) - - rcpt, err := someotherreader.Read(concludeCav.Receipt, ucanConcludeMsg.Blocks()) - require.NoError(t, err) - - // get the transfer caveats and assert they match expected values - transferCav := mustGetInvocationCaveats[replica.TransferCaveats]( - t, reader, rcpt.Ran().Link(), - replica.TransferCaveatsReader.Read, - ) - require.EqualValues(t, expectedSize, transferCav.Blob.Size) - require.Equal(t, expectedDigest, transferCav.Blob.Digest) - require.Equal(t, expectedSpace, transferCav.Space) - - // extract the location claim from the transfer invocation - locationCav := mustGetInvocationCaveats[assert.LocationCaveats]( - t, reader, transferCav.Site, assert.LocationCaveatsReader.Read, - ) - require.Equal(t, expectedLocationCav, locationCav) - - // verify cause -> points back to replica allocate - replicaAllocateCav := mustGetInvocationCaveats[replica.AllocateCaveats]( - t, reader, transferCav.Cause, replica.AllocateCaveatsReader.Read, - ) - require.Equal(t, expectedAllocateCav, replicaAllocateCav) - - // verify replica allocate cause is blob replicate - blobReplicateCav := mustGetInvocationCaveats[blob2.ReplicateCaveats]( - t, reader, replicaAllocateCav.Cause, blob2.ReplicateCaveatsReader.Read, - ) - require.Equal(t, expectedReplicaCav, blobReplicateCav) - - // read the transfer receipt - transferReceiptCid := testutil.Must( - cid.Parse(rcpt.Root().Link().String()), - )(t) - transferReceiptReader := testutil.Must( - receipt.NewReceiptReaderFromTypes[replica.TransferOk, failure.FailureModel]( - replica.TransferOkType(), failure.FailureType(), types.Converters..., - ), - )(t) - transferReceipt := testutil.Must( - transferReceiptReader.Read(cidlink.Link{Cid: transferReceiptCid}, reader.Iterator()), - )(t) - transferOk := testutil.Must( - result.Unwrap(result.MapError(transferReceipt.Out(), failure.FromFailureModel)), - )(t) - - // PDP isn't enabled in this test setup, so no PDP proof expected. - require.Nil(t, transferOk.PDP) - - // read the receipt of the transfer invocation asserting the location caveats of Site contain expected values. - locationCavRct := mustGetInvocationCaveats[assert.LocationCaveats](t, reader, transferOk.Site, assert.LocationCaveatsReader.Read) - require.Equal(t, expectedSpace, locationCavRct.Space) - require.Equal(t, expectedDigest, locationCavRct.Content.Hash()) - require.Len(t, locationCavRct.Location, 1) - require.Equal(t, fmt.Sprintf("/blob/z%s", expectedDigest.B58String()), locationCavRct.Location[0].Path) - -} - -// Reads the final “transfer invocation” and asserts its fields, and chain of invocations -func mustAssertTransferInvocation( - t *testing.T, - ucanConcludeMsg message.AgentMessage, - expectedDigest multihash.Multihash, - expectedSize uint64, - expectedSpace did.DID, - expectedLocationCav assert.LocationCaveats, - expectedAllocateCav replica.AllocateCaveats, - expectedReplicaCav blob2.ReplicateCaveats, -) { - // sanity check - require.NotNil(t, ucanConcludeMsg) - - concludeInvocationCid := testutil.Must( - cid.Parse(ucanConcludeMsg.Invocations()[0].String()), - )(t) - reader := testutil.Must( - blockstore.NewBlockReader(blockstore.WithBlocksIterator(ucanConcludeMsg.Blocks())), - )(t) - - concludeCav := mustGetInvocationCaveats[ucancap.ConcludeCaveats]( - t, reader, cidlink.Link{Cid: concludeInvocationCid}, - ucancap.ConcludeCaveatsReader.Read, - ) - someotherreader, err := receipt.NewReceiptReaderFromTypes[replica.TransferOk, failure.FailureModel](replica.TransferOkType(), failure.FailureType(), types.Converters...) - require.NoError(t, err) - - rcpt, err := someotherreader.Read(concludeCav.Receipt, ucanConcludeMsg.Blocks()) - require.NoError(t, err) - - // get the transfer caveats and assert they match expected values - transferCav := mustGetInvocationCaveats[replica.TransferCaveats]( - t, reader, rcpt.Ran().Link(), - replica.TransferCaveatsReader.Read, - ) - require.EqualValues(t, expectedSize, transferCav.Blob.Size) - require.Equal(t, expectedDigest, transferCav.Blob.Digest) - require.Equal(t, expectedSpace, transferCav.Space) - - // extract the location claim from the transfer invocation - locationCav := mustGetInvocationCaveats[assert.LocationCaveats]( - t, reader, transferCav.Site, assert.LocationCaveatsReader.Read, - ) - require.Equal(t, expectedLocationCav, locationCav) - - // verify cause -> points back to replica allocate - replicaAllocateCav := mustGetInvocationCaveats[replica.AllocateCaveats]( - t, reader, transferCav.Cause, replica.AllocateCaveatsReader.Read, - ) - require.Equal(t, expectedAllocateCav, replicaAllocateCav) - - // verify replica allocate cause is blob replicate - blobReplicateCav := mustGetInvocationCaveats[blob2.ReplicateCaveats]( - t, reader, replicaAllocateCav.Cause, blob2.ReplicateCaveatsReader.Read, - ) - require.Equal(t, expectedReplicaCav, blobReplicateCav) - - // read the transfer receipt - transferReceiptCid := testutil.Must( - cid.Parse(rcpt.Root().Link().String()), - )(t) - transferReceiptReader := testutil.Must( - receipt.NewReceiptReaderFromTypes[replica.TransferOk, failure.FailureModel]( - replica.TransferOkType(), failure.FailureType(), types.Converters..., - ), - )(t) - transferReceipt := testutil.Must( - transferReceiptReader.Read(cidlink.Link{Cid: transferReceiptCid}, reader.Iterator()), - )(t) - transferOk := testutil.Must( - result.Unwrap(result.MapError(transferReceipt.Out(), failure.FromFailureModel)), - )(t) - - // PDP isn't enabled in this test setup, so no PDP proof expected. - require.Nil(t, transferOk.PDP) - - // read the receipt of the transfer invocation asserting the location caveats of Site contain expected values. - locationCavRct := mustGetInvocationCaveats[assert.LocationCaveats](t, reader, transferOk.Site, assert.LocationCaveatsReader.Read) - require.Equal(t, expectedSpace, locationCavRct.Space) - require.Equal(t, expectedDigest, locationCavRct.Content.Hash()) - require.Len(t, locationCavRct.Location, 1) - require.Equal(t, fmt.Sprintf("/blob/z%s", expectedDigest.B58String()), locationCavRct.Location[0].Path) -} - -func mustGetInvocationCaveats[T ipld.Builder](t *testing.T, reader blockstore.BlockReader, inv ucan.Link, invReader func(any) (T, ufailure.Failure)) T { - view := testutil.Must(invocation.NewInvocationView(inv, reader))(t) - invc := testutil.Must(invReader(view.Capabilities()[0].Nb()))(t) - return invc -} - -// startTestHTTPServer starts a simple HTTP server with configurable endpoints. -func startTestHTTPServer( - ctx context.Context, - t *testing.T, - digest multihash.Multihash, - serveData []byte, - svc Service, - addr, sourcePath, sinkPath, uploadServicePath string, -) (*http.Server, <-chan message.AgentMessage) { - agentCh := make(chan message.AgentMessage, 1) - mux := http.NewServeMux() - - // Endpoint to serve data. - mux.HandleFunc(fmt.Sprintf("/%s", sourcePath), func(w http.ResponseWriter, r *http.Request) { - req := ucan_http.NewRequest(r.Body, r.Header) - switch r.Method { - case http.MethodGet: // UCAN authorized retrieval for a blob - codec := headercar.NewInboundCodec() - accept := testutil.Must(codec.Accept(req))(t) - msg := testutil.Must(accept.Decoder().Decode(req))(t) - bs := testutil.Must(blockstore.NewBlockReader(blockstore.WithBlocksIterator(msg.Blocks())))(t) - inv := testutil.Must(invocation.NewInvocationView(msg.Invocations()[0], bs))(t) - // this also works for blob/retrieve since result is empty obj - out := result.Ok[content.RetrieveOk, ipld.Builder](content.RetrieveOk{}) - // alice is hard coded in the location claim so it is also hard coded here - rcpt := testutil.Must(receipt.Issue(testutil.Alice, out, ran.FromInvocation(inv)))(t) - msg = testutil.Must(message.Build(nil, []receipt.AnyReceipt{rcpt}))(t) - resp := testutil.Must(accept.Encoder().Encode(msg))(t) - for key, values := range resp.Headers() { - for _, val := range values { - w.Header().Add(key, val) - } - } - _, _ = w.Write(serveData) - case http.MethodPost: // UCAN invocation for access/grant - codec := ucan_car.NewInboundCodec() - accept := testutil.Must(codec.Accept(req))(t) - msg := testutil.Must(accept.Decoder().Decode(req))(t) - bs := testutil.Must(blockstore.NewBlockReader(blockstore.WithBlocksIterator(msg.Blocks())))(t) - inv := testutil.Must(invocation.NewInvocationView(msg.Invocations()[0], bs))(t) - cap := inv.Capabilities()[0] - if cap.Can() != access.GrantAbility { - t.Fatal("unexpected invocation") - } - dlg := testutil.Must(delegation.Delegate( - testutil.Alice, - inv.Issuer(), - []ucan.Capability[ucan.NoCaveats]{ - ucan.NewCapability("*", testutil.Alice.DID().String(), ucan.NoCaveats{}), - }, - ))(t) - dlgsModel := access.DelegationsModel{ - Keys: []string{dlg.Link().String()}, - Values: map[string][]byte{ - dlg.Link().String(): testutil.Must(io.ReadAll(dlg.Archive()))(t), - }, - } - out := result.Ok[access.GrantOk, ipld.Builder](access.GrantOk{Delegations: dlgsModel}) - // alice is hard coded in the location claim so it is also hard coded here - rcpt, err := receipt.Issue(testutil.Alice, out, ran.FromInvocation(inv)) - require.NoError(t, err) - msg, err = message.Build(nil, []receipt.AnyReceipt{rcpt}) - require.NoError(t, err) - resp, err := accept.Encoder().Encode(msg) - require.NoError(t, err) - for key, values := range resp.Headers() { - for _, val := range values { - w.Header().Add(key, val) - } - } - _ = testutil.Must(io.Copy(w, resp.Body()))(t) - default: - t.Fatal("unexpected invocation") - } - }) - // Endpoint to store data on the replica. - mux.HandleFunc(fmt.Sprintf("/%s", sinkPath), func(w http.ResponseWriter, r *http.Request) { - require.NoError(t, svc.Blobs().Store().Put(ctx, digest, uint64(len(serveData)), bytes.NewReader(serveData))) - _, _ = w.Write(serveData) - }) - // Endpoint to simulate the upload service. - mux.HandleFunc(fmt.Sprintf("/%s", uploadServicePath), func(w http.ResponseWriter, r *http.Request) { - roots, blocks, err := car.Decode(r.Body) - require.NoError(t, err) - bstore, err := blockstore.NewBlockReader(blockstore.WithBlocksIterator(blocks)) - require.NoError(t, err) - agentMessage, err := message.NewMessage(roots[0], bstore) - require.NoError(t, err) - agentCh <- agentMessage - }) - - server := &http.Server{ - Addr: addr, - Handler: mux, - } - - var listenErr error - go func() { - if err := server.ListenAndServe(); err != nil { - listenErr = err - } - }() - time.Sleep(500 * time.Millisecond) - require.NoError(t, listenErr) - t.Cleanup(func() { - require.NoError(t, server.Close()) - }) - return server, agentCh -} - -// FakePresigned is a stub for upload URL presigning. -// TODO turn this into a mock -type FakePresigned struct { - uploadURL url.URL -} - -func (f *FakePresigned) SignUploadURL(ctx context.Context, digest multihash.Multihash, size, ttl uint64) (url.URL, http.Header, error) { - return f.uploadURL, nil, nil -} - -func (f *FakePresigned) VerifyUploadURL(ctx context.Context, url url.URL, headers http.Header) (url.URL, http.Header, error) { - // TODO: implement when needed. - panic("implement me") -}