diff --git a/.cursor/rules/no-credentials-in-docs.mdc b/.cursor/rules/no-credentials-in-docs.mdc new file mode 100644 index 0000000..359b37d --- /dev/null +++ b/.cursor/rules/no-credentials-in-docs.mdc @@ -0,0 +1,50 @@ +--- +description: Prevent credentials and secrets in committed documentation +alwaysApply: true +--- + +# No credentials in documentation + +When creating or editing files that may be committed (especially `*.md`, `*.mdx`, `*.example`, `README*`, `NOTES*`, `docs/**`), **never** embed real secrets. + +## Do not include + +- Passwords, passphrases, API keys, tokens, cookies, private keys, WIFs +- Basic-auth pairs in curl/examples (`-u realuser:realpassword`) +- Values copied from the user's `.env`, `NOTES.md`, `setup_*.sh`, Docker `-e` flags, or chat history +- Production hostnames/IPs tied to identifiable infra (prefer `127.0.0.1`, `localhost`, `example.com`) + +## Use placeholders instead + +```bash +# ❌ BAD — real credential from local setup +curl -u subs:NeverASecret! http://127.0.0.1:7777/spaces/@space/pipeline + +# ✅ GOOD — env vars or generic placeholders +curl -u "$SUBS_BASIC_AUTH_USER:$SUBS_BASIC_AUTH_PASSWORD" "$BASE/spaces/$SPACE/pipeline" + +# ✅ GOOD — document optional auth without values +# If basic auth is enabled: -u "$SUBS_BASIC_AUTH_USER:$SUBS_BASIC_AUTH_PASSWORD" +curl "$BASE/spaces/$SPACE/pipeline" +``` + +```bash +# ❌ BAD +-e SUBS_BASIC_AUTH_PASSWORD=Whatever84 + +# ✅ GOOD +-e SUBS_BASIC_AUTH_PASSWORD=change-me +# or reference .env.example without real values +``` + +## Before finishing doc changes + +1. Search the diff for patterns: `password`, `secret`, `token`, `api_key`, `-u `, `Bearer `, `BEGIN.*PRIVATE KEY`, `xprv`, `WIF` +2. Replace any value that came from the user's environment with a placeholder or env-var reference +3. If a real secret was already written, redact it and tell the user to rotate the credential + +## Safe sources + +- `.env.example` with obvious placeholders (`change-me`, `your-password-here`) +- Variable **names** and config keys (e.g. `SUBS_SPACED_RPC_PASSWORD`) without real values +- Public URLs documented by the project (e.g. fabric relay seeds) diff --git a/.dockerignore b/.dockerignore index f7eab8f..e2b03aa 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,13 +11,21 @@ target # Local operator state, never part of an image. data testrig-data +datamad *.bin *.priv *.subs *.subs.c *.sdb +# Secrets and local notes (keep README for image docs if needed). +.env +.env.* +NOTES.md +*.md +!README.md + .DS_Store **/.DS_Store screenshot.png -LICENSE \ No newline at end of file +LICENSE diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..2a232c9 --- /dev/null +++ b/.env.example @@ -0,0 +1,48 @@ +# Copy to .env and adjust, or export variables in your shell. +# CLI flags take precedence over environment variables. + +# --- subs --- +SUBS_PORT=7777 +SUBS_DATA_DIR=./data +SUBS_WALLET=my-wallet +SUBS_SPACED_RPC_URL=http://127.0.0.1:7225 +# SUBS_SPACED_RPC_USER=testuser +# SUBS_SPACED_RPC_PASSWORD=secret +# SUBS_SPACED_RPC_COOKIE=/path/to/.cookie +# Optional HTTP Basic auth for the UI/API. Set BOTH to enable (GET /health stays anonymous). +# SUBS_BASIC_AUTH_USER=admin +# SUBS_BASIC_AUTH_PASSWORD=change-me +# In Docker, subs-prover and registry-server run in the same container as subs by default. +SUBS_PROVER_ENDPOINT=http://127.0.0.1:8888 +SUBS_REGISTRY_ENDPOINT=http://127.0.0.1:8081 +# SUBS_START_PROVER=1 +# SUBS_START_REGISTRY=1 +# SUBS_PROVER_SERVER=1 +# SUBS_ENV_FILE=.env +# SUBS_TEST_RIG=1 +# Block publish until commitments reach 150 on-chain confirmations. +# Default: false (unset, empty, 0, false, no, off). Truthy: 1, true, yes, on. +# SUBS_PUBLISH_REQUIRE_FINALIZED=1 +# SUBS_TEST_RIG_DIR=./testrig-data + +# --- subs-prover --- +# SUBS_PROVER_SERVER=1 +SUBS_PROVER_PORT=8888 +# SUBS_PROVER_ENV_FILE=.env +# SUBS_PROVER_INPUT=request.json +# SUBS_PROVER_OUTPUT=receipt.bin +# Optional bearer auth for the prover (Authorization: Bearer ). +# When set, all routes including /health require the token. +# PROVER_AUTH_TOKEN=change-me +# Opt-in startup calibration (blocks listen until done). Also --calibrate. +# PROVER_CALIBRATE=1 + +# --- registry-server (example) --- +REGISTRY_SERVER_PORT=8081 +# Both keys are required and must differ (see examples/registry-server/README.md). +# REGISTRY_API_KEY=change-me-intake +# SUBSD_API_KEY=change-me-subsd +# REGISTRY_SERVER_ENV_FILE=.env + +# --- logging (all components) --- +# RUST_LOG=subs=info,subs_prover=info,registry_server=info,tower_http=debug diff --git a/.gitignore b/.gitignore index 76d0660..513ec67 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +.env +.env.* +!.env.example .DS_Store .vscode target/ @@ -8,4 +11,9 @@ target/ *.sdb .idea testrig-data -data +data/ +datamad/ +datamadd/ +NOTES.md +.cargo/ +subspaces.tar diff --git a/Cargo.lock b/Cargo.lock index 1041533..5edc49d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1593,6 +1593,14 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "config-origins" +version = "0.1.0" +dependencies = [ + "clap", + "dotenvy", +] + [[package]] name = "console" version = "0.15.11" @@ -2099,6 +2107,12 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf673e0848ef09fa4aeeba78e681cf651c0c7d35f76ee38cec8e55bc32fa111" +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "downcast-rs" version = "1.2.1" @@ -4907,6 +4921,7 @@ dependencies = [ "anyhow", "axum 0.7.9", "clap", + "config-origins", "serde", "serde_json", "tokio", @@ -6392,6 +6407,8 @@ dependencies = [ "bitcoin", "borsh", "clap", + "config-origins", + "dotenvy", "hex", "relay", "reqwest", @@ -6447,6 +6464,8 @@ dependencies = [ "axum 0.7.9", "borsh", "clap", + "config-origins", + "dotenvy", "libveritas", "libveritas_zk", "risc0-zkvm", @@ -7372,6 +7391,16 @@ dependencies = [ "safe_arch", ] +[[package]] +name = "wif-to-hex" +version = "0.1.0" +dependencies = [ + "anyhow", + "bitcoin", + "clap", + "hex", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 05cb984..0eaea62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,14 @@ [workspace] resolver = "2" -members = ["core", "prover", "types", "subs", "examples/registry-server"] +members = [ + "config-origins", + "core", + "prover", + "types", + "subs", + "examples/registry-server", + "tools/wif-to-hex", +] [workspace.dependencies] # Internal crates @@ -31,7 +39,8 @@ serde_json = "1.0" anyhow = "1.0" hex = "0.4" borsh = { version = "1.5", default-features = false, features = ["derive"] } -clap = { version = "4.5", features = ["derive"] } +clap = { version = "4.5", features = ["derive", "env"] } +dotenvy = "0.15" tokio = { version = "1" } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..78bd9fc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,132 @@ +# syntax=docker/dockerfile:1 + +# Rust toolchain on Alpine (musl) for release binaries. +FROM rust:1-alpine3.21 AS builder-base + +ARG CARGO_BUILD_JOBS=1 + +RUN apk add --no-cache \ + build-base \ + musl-dev \ + git \ + openssl-dev \ + openssl-libs-static \ + pkgconf \ + clang \ + llvm-dev \ + lld \ + libatomic \ + ca-certificates + +WORKDIR /app +ENV CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS} +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +ENV RUSTFLAGS="-C link-arg=-fuse-ld=lld" + +# subs + registry share a target/ tree (small compared to subs-prover). +FROM builder-base AS builder-subs + +ARG ENABLE_REGISTRY=true + +COPY . . + +RUN set -eux; \ + cargo build --release -p subs; \ + if [ "$ENABLE_REGISTRY" != "false" ]; then \ + cargo build --release -p registry-server; \ + fi; \ + mkdir -p /out; \ + cp target/release/subs /out/; \ + if [ "$ENABLE_REGISTRY" != "false" ]; then \ + cp target/release/registry-server /out/; \ + fi; \ + cargo clean + +# subs-prover (RISC Zero) in a fresh stage so target/ does not stack on top of subs. +FROM builder-base AS builder-prover + +ARG ENABLE_PROVER=true +ARG GPU_ACCELERATION=none +ARG TARGETARCH + +COPY . . + +RUN set -eux; \ + if [ "$ENABLE_PROVER" = "false" ]; then \ + mkdir -p /out; \ + exit 0; \ + fi; \ + if [ "$TARGETARCH" = "arm64" ]; then \ + export CFLAGS="-mno-outline-atomics"; \ + export CXXFLAGS="-mno-outline-atomics"; \ + export CMAKE_C_FLAGS="-mno-outline-atomics"; \ + export CMAKE_CXX_FLAGS="-mno-outline-atomics"; \ + export RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-feature=-outline-atomics"; \ + else \ + export RUSTFLAGS="-C link-arg=-fuse-ld=lld"; \ + fi; \ + case "$GPU_ACCELERATION" in \ + none) cargo build --release -p subs-prover ;; \ + metal) cargo build --release -p subs-prover --features metal ;; \ + cuda) cargo build --release -p subs-prover --features cuda ;; \ + *) echo "Invalid GPU_ACCELERATION=$GPU_ACCELERATION (expected none, metal, or cuda)" >&2; exit 1 ;; \ + esac; \ + mkdir -p /out; \ + cp target/release/subs-prover /out/; \ + cargo clean + +FROM alpine:3.21 + +ARG ENABLE_PROVER=true +ARG ENABLE_REGISTRY=true +ARG GPU_ACCELERATION=none + +RUN apk add --no-cache ca-certificates libgcc tini \ + && addgroup -S subs \ + && adduser -S subs -G subs + +COPY --from=builder-subs /out/subs /usr/local/bin/subs + +RUN --mount=type=bind,from=builder-subs,source=/out,target=/subs-out \ + --mount=type=bind,from=builder-prover,source=/out,target=/prover-out \ + set -eux; \ + if [ "$ENABLE_REGISTRY" != "false" ] && [ -f /subs-out/registry-server ]; then \ + cp /subs-out/registry-server /usr/local/bin/registry-server; \ + fi; \ + if [ "$ENABLE_PROVER" != "false" ] && [ -f /prover-out/subs-prover ]; then \ + cp /prover-out/subs-prover /usr/local/bin/subs-prover; \ + fi; \ + : > /etc/subs-image.env; \ + if [ "$ENABLE_PROVER" != "false" ]; then \ + echo "SUBS_START_PROVER=1" >> /etc/subs-image.env; \ + echo "SUBS_PROVER_ENDPOINT=http://127.0.0.1:8888" >> /etc/subs-image.env; \ + echo "SUBS_PROVER_SERVER=1" >> /etc/subs-image.env; \ + echo "SUBS_PROVER_GPU_ACCELERATION=${GPU_ACCELERATION}" >> /etc/subs-image.env; \ + else \ + echo "SUBS_START_PROVER=0" >> /etc/subs-image.env; \ + fi; \ + if [ "$ENABLE_REGISTRY" != "false" ]; then \ + echo "SUBS_START_REGISTRY=1" >> /etc/subs-image.env; \ + echo "SUBS_REGISTRY_ENDPOINT=http://127.0.0.1:8081" >> /etc/subs-image.env; \ + else \ + echo "SUBS_START_REGISTRY=0" >> /etc/subs-image.env; \ + fi + +COPY docker/entrypoint.sh /entrypoint.sh + +RUN chmod +x /entrypoint.sh \ + && mkdir -p /data \ + && chown -R subs:subs /data + +WORKDIR /data +USER subs + +ENV SUBS_DATA_DIR=/data +ENV SUBS_PORT=7777 +ENV SUBS_PROVER_PORT=8888 +ENV REGISTRY_SERVER_PORT=8081 + +EXPOSE 7777 8888 8080 8081 + +ENTRYPOINT ["/sbin/tini", "--", "/entrypoint.sh"] +CMD ["subs"] diff --git a/FIX_SIGNATURE_INVALID.md b/FIX_SIGNATURE_INVALID.md new file mode 100644 index 0000000..a70d908 --- /dev/null +++ b/FIX_SIGNATURE_INVALID.md @@ -0,0 +1,190 @@ +# Fixing "signature invalid" / Unresolvable Subhandles + +Symptom: `@space` resolves from the chain anchor alone, but a subhandle like `gxxxxxx@space` fails to resolve (e.g. `receipt required for @space`), and/or publish returns `rejected: signature invalid for @space`. + +The failure means relays have enough data to anchor `@space` at the chain level, but not a **complete, verifiable certificate chain** for subhandles. For commitment index ≥ 1, the **root cert must include a ZK receipt**; without it, `libveritas` rejects the bundle and the subhandle cannot resolve. + +Fix it by finishing the pipeline, then **republishing**. + +--- + +## 1. Diagnose where you are stuck + +Open the space page for `@space` and check the pipeline stepper, or use the API: + +```bash +BASE=http://127.0.0.1:7777 +SPACE=%40space # URL-encoded @space + +# If basic auth is enabled, prefix curl with: +# -u "$SUBS_BASIC_AUTH_USER:$SUBS_BASIC_AUTH_PASSWORD" + +curl -s "$BASE/spaces/$SPACE/pipeline" | jq +curl -s "$BASE/spaces/$SPACE/commit/status" | jq +curl -s "$BASE/spaces/$SPACE/handles?filter=unpublished" | jq +curl -s "$BASE/spaces/$SPACE/handles/gxxxxxx" | jq +``` + +Look for: + +| Signal | Meaning | +|--------|---------| +| `current_step: "proving"` | STARK proof not done — **cannot publish valid root cert** | +| `current_step: "broadcast"` | Committed locally but not on-chain | +| `confirmed` but not `finalized` | On-chain but UI still waiting (publish is OK once **Confirmed**) | +| `publish_status: null` | Never successfully published | +| `publish_status: "temp"` | Published before chain caught up — may need republish | +| Space shows **STARK** badge | Receipt exists locally | +| **Untracked On-Chain Commitment** warning | Local DB out of sync with chain — fix data dir / sync first | + +Also check the handle’s `commitment_idx` vs on-chain tip: if the handle is committed locally but not yet at chain tip, publish will issue **temp** certs that veritas may not accept the same way. + +--- + +## 2. Complete the pipeline (in order) + +The full sequence is documented in `SUBS_PUBLISH.md`: + +``` +Stage → Local commit → [Proving] → Broadcast → Confirmed → Publish → Resolve +``` + +### If handles are still staged + +**Commit Local** (UI) or: + +```bash +curl -X POST "$BASE/spaces/$SPACE/commit" \ + -H "Content-Type: application/json" -d '{"dry_run":false}' +``` + +### If this is commitment #2 or later (`is_initial: false`) + +You **must** prove before broadcast. Root certs embed the receipt from local DB: + +```bash +# Prover must be running and SUBS_PROVER_ENDPOINT reachable +curl -X POST "$BASE/spaces/$SPACE/proving/push" +curl -X POST "$BASE/spaces/$SPACE/proving/poll" +``` + +Repeat poll until proving step is **Complete**. Verify: + +```bash +curl -s "$BASE/spaces/$SPACE/proving/next" +# should return empty / no pending request +``` + +Without a stored step (and fold, for idx ≥ 2) receipt, `issue_cert` for `@space` cannot attach the receipt libveritas requires. + +### Broadcast on-chain + +```bash +curl -X POST "$BASE/spaces/$SPACE/broadcast" \ + -H "Content-Type: application/json" -d '{"fee_rate": 1.0}' +``` + +Wait until `commit/status` shows **Confirmed** (on-chain tip root matches your commitment). **Do not publish before this** — early publish produces temp certs that relays often reject (`signature invalid for @space`). + +### Publish certificates + +Publish the subhandle (subs includes the root cert in the bundle): + +```bash +curl -X POST "$BASE/spaces/$SPACE/publish" \ + -H "Content-Type: application/json" \ + -d '{"handles":["gxxxxxx"]}' +``` + +Or use **Publish** on the space page. Subs will: + +1. Reset stale temp certs if chain tip moved +2. Issue root cert **with receipt** + leaf cert for `gxxxxxx@space` +3. Broadcast to fabric relays + +Success looks like: + +```json +{ "handles_published": 1, "remaining": 0 } +``` + +Handle should show `publish_status: "final"` (not `temp`) once its commitment is confirmed on-chain. + +--- + +## 3. Verify resolution + +```bash +curl -X POST "$BASE/query" \ + -H "Content-Type: application/json" \ + -d '{"handle":"gxxxxxx@space"}' +``` + +Or use the Query UI. You want a zone back with script pubkey and records, not a verify error. + +CLI equivalent: + +```bash +fabric resolve gxxxxxx@space +``` + +--- + +## 4. If you already published but resolve still fails + +**Republish after fixing upstream steps:** + +1. Confirm proving is complete and commitment is **Confirmed** on-chain. +2. Republish the handle (UI: **Re-publish Certificate**, or same `POST /publish` curl). +3. `publish_certs` automatically calls `reset_stale_temp_certs` when the chain tip changed, clearing old temp publishes so they get reissued. + +If publish returns `signature invalid for ...`: + +- Commitment not yet at chain tip, or +- Wrong operating wallet, or +- Stale temp cert — wait for confirm, then republish. + +If publish succeeds but resolve still says `receipt required for @space`: + +- Root cert on relays is still old/incomplete — republish after proving receipts exist locally (check for **STARK** badge on space page). +- Relays may need a moment to propagate; retry resolve after a successful publish. + +--- + +## 5. Docker-specific checks + +Ensure inside the container: + +- `SUBS_PROVER_ENDPOINT` points to a **running** prover (host `127.0.0.1:8888` from inside Docker is the container itself — use host IP or run prover in the same container). +- `SUBS_SPACED_RPC_URL` reaches your spaced node. +- `SUBS_DATA_DIR` is the **same** data directory where commits and receipts were stored (wrong data dir → missing receipts → publish without receipt). + +--- + +## Quick decision tree + +``` +Is commitment idx >= 1? + ├─ Yes → Is proving complete? (step/fold receipts in DB) + │ ├─ No → Prove first, then broadcast, then publish + │ └─ Yes → Is commit on-chain (Confirmed)? + │ ├─ No → Broadcast, wait for confirm, then publish + │ └─ Yes → Publish (or republish) gxxxxxx + └─ No (genesis only) → Broadcast if needed, then publish +``` + +The root cause is almost always: **subhandle publish never completed with a root cert that includes the STARK receipt**, because proving, broadcast, or publish was incomplete or done out of order. Walk the pipeline on `@space` until every step is green, then republish `gxxxxxx`. + +--- + +## Related code + +| Concern | Location | +|---------|----------| +| Publish flow | `core/src/app.rs` (`publish_certs`, `issue_certs`, `issue_cert`) | +| Root cert + receipt | `core/src/core.rs` (`issue_cert`, `get_receipt`) | +| Stale temp reset | `core/src/storage.rs` (`reset_stale_temp_certs`) | +| Unpublished selection | `core/src/storage.rs` (`HandleSelector::Unpublished`) | +| Pipeline status | `core/src/app.rs` (`get_pipeline_status`) | +| Resolve/verify | `core/src/app.rs` (`resolve`) + `fabric-resolver` / `libveritas` | +| Full walkthrough | `SUBS_PUBLISH.md`, `VERITAS_RESOLUTION.md` | diff --git a/README.md b/README.md index 13d6e86..c68f662 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,252 @@ cargo install --path prover For operators, use `--features cuda` on `subs-prover` for nvidia machines to enable GPU acceleration. +## Configuration + +Each binary accepts the same settings via **CLI flags**, **environment variables**, or a **`.env` file** in the current working directory. Command-line flags override environment variables. + +Load a custom env file path with: + +- `subs`: `SUBS_ENV_FILE=/path/to/subs.env` +- `subs-prover`: `SUBS_PROVER_ENV_FILE=/path/to/prover.env` +- `registry-server`: `REGISTRY_SERVER_ENV_FILE=/path/to/registry.env` + +See [.env.example](.env.example) for a full template. + +### `subs` + +| Variable | CLI flag | Description | +|----------|----------|-------------| +| `SUBS_PORT` | `--port` | HTTP server port (default `7777`) | +| `SUBS_DATA_DIR` | `--data-dir` | Data directory (default `./data`) | +| `SUBS_WALLET` | `--wallet` | Wallet name for signing | +| `SUBS_SPACED_RPC_URL` | `--rpc-url` | `spaced` RPC URL | +| `SUBS_SPACED_RPC_USER` | `--rpc-user` | `spaced` RPC username | +| `SUBS_SPACED_RPC_PASSWORD` | `--rpc-password` | `spaced` RPC password | +| `SUBS_SPACED_RPC_COOKIE` | `--rpc-cookie` | `spaced` RPC cookie file path | +| `SUBS_PROVER_ENDPOINT` | *(Settings UI)* | Prover URL written to `config.db` at startup | +| `SUBS_REGISTRY_ENDPOINT` | *(Settings UI)* | Registry URL written to `config.db` at startup | +| `SUBS_BASIC_AUTH_USER` / `SUBS_BASIC_AUTH_PASSWORD` | `--basic-auth-user` / `--basic-auth-password` | Optional HTTP Basic for UI/API (both required; `/health` stays anonymous) | +| `SUBS_PUBLISH_REQUIRE_FINALIZED` | `--publish-require-finalized` | Block publish until commitments have 150 confirmations | +| `SUBS_TEST_RIG` | `--test-rig` | Enable test rig (`1`, `true`, `yes`) | +| `SUBS_TEST_RIG_DIR` | `--test-rig-dir` | Test rig data directory | + +### `subs-prover` + +| Variable | CLI flag | Description | +|----------|----------|-------------| +| `SUBS_PROVER_SERVER` | `--server` | Run as HTTP server (`1`, `true`, `yes`) | +| `SUBS_PROVER_PORT` | `--server-port` | Server port (default `8888`) | +| `PROVER_AUTH_TOKEN` | *(env only)* | Optional bearer token; when set, all prover routes require `Authorization: Bearer ` | +| `PROVER_CALIBRATE` | `--calibrate` | Opt-in startup calibration (blocks listen until done; needed for `/estimate`) | +| `SUBS_PROVER_INPUT` | `-i` / `--input` | Input file (prove/compress subcommands) | +| `SUBS_PROVER_OUTPUT` | `-o` / `--output` | Output file (prove/compress subcommands) | +| `SUBS_PROVER_BENCH_EXISTING` | `--existing` | Bench: existing handle count | +| `SUBS_PROVER_BENCH_INSERT` | `--insert` | Bench: handles to insert | + +### `registry-server` + +| Variable | CLI flag | Description | +|----------|----------|-------------| +| `REGISTRY_SERVER_PORT` | `--port` | HTTP server port (default `8081`) | +| `REGISTRY_API_KEY` | *(env only, required)* | Bearer token for `POST /register` | +| `SUBSD_API_KEY` | *(env only, required)* | Bearer token for subs↔registry endpoints (`/health`, `/pending`, `/ack`, `/committed`) | + +### Examples + +Using `export`: + +```bash +export SUBS_SPACED_RPC_URL=http://127.0.0.1:7225 +export SUBS_WALLET=my-wallet +export SUBS_DATA_DIR=./data +export SUBS_PROVER_ENDPOINT=http://127.0.0.1:8888 +subs +``` + +Using a `.env` file: + +```bash +cp .env.example .env +# edit .env, then: +subs +``` + +```bash +# subs-prover from .env +export SUBS_PROVER_SERVER=1 +export SUBS_PROVER_PORT=8888 +subs-prover +``` + +```bash +# registry-server +export REGISTRY_SERVER_PORT=8081 +export REGISTRY_API_KEY=change-me-intake +export SUBSD_API_KEY=change-me-subsd +registry-server +``` + +Log verbosity uses the standard `RUST_LOG` variable (e.g. `RUST_LOG=subs=debug,tower_http=debug`). + +On startup, each binary prints its **effective configuration** to the console with the **origin** of each value: `param` (CLI flag), `environment` (`export`), `.env` (dotenv file), or `default`. Sensitive values (passwords) are shown as `(set)` without revealing the secret. Example: + +``` +subs configuration: + (loaded env file: .env) + port = 7777 (.env) + data_dir = ./datamad (.env) + wallet = mad (environment) + rpc_url = http://127.0.0.1:7225 (.env) + rpc_password = (set) (.env) + server_url = http://127.0.0.1:7777 (derived from port) +``` + +CLI flags override environment variables; process environment overrides `.env` for the same key. + +## Docker + +The image is built from **Rust on Alpine** (musl). By default it includes `subs`, `subs-prover`, and `registry-server`; use build args to omit optional components. An entrypoint dispatches by component name or `SUBS_COMPONENT`. + +When all components are included, starting `subs` also starts **subs-prover** and **registry-server** in the same container: + +| Service | Default URL | Disable with | +|---------|-------------|--------------| +| subs-prover | `http://127.0.0.1:8888` (`SUBS_PROVER_ENDPOINT`) | `SUBS_START_PROVER=0` | +| registry-server | `http://127.0.0.1:8081` (`SUBS_REGISTRY_ENDPOINT`) | `SUBS_START_REGISTRY=0` | + +**Note:** The image build includes RISC Zero proving when `ENABLE_PROVER` is enabled; use `GPU_ACCELERATION` to select CPU (`none`), Apple Metal (`metal`), or NVIDIA CUDA (`cuda`) for `subs-prover`. + +### Build + +Full image (subs + prover + registry): + +```bash +docker build -t subs:latest . +``` + +Subs only (faster build; skips RISC Zero prover and registry): + +```bash +docker build -t subs:slim \ + --build-arg ENABLE_PROVER=false \ + --build-arg ENABLE_REGISTRY=false . +``` + +Omit only the prover: + +```bash +docker build -t subs:no-prover --build-arg ENABLE_PROVER=false . +``` + +Omit only the registry: + +```bash +docker build -t subs:no-registry --build-arg ENABLE_REGISTRY=false . +``` + +Build args: + +| Build arg | Default | Description | +|-----------|---------|-------------| +| `ENABLE_PROVER` | `true` | Set to `false` to skip building/shipping `subs-prover` | +| `ENABLE_REGISTRY` | `true` | Set to `false` to skip building/shipping `registry-server` | +| `GPU_ACCELERATION` | `none` | `subs-prover` features: `none` (CPU), `metal`, or `cuda` | +| `CARGO_BUILD_JOBS` | `1` | Parallel `rustc` jobs in the builder (raise only if Docker has enough RAM) | + +**Memory:** A full image with `subs-prover` often needs **8 GB+** RAM for the Docker builder VM. If the build fails with `cannot allocate memory`, increase **Docker Desktop → Settings → Resources → Memory**, keep `CARGO_BUILD_JOBS=1` (default), or build without the prover. + +**Disk:** `subs-prover` (RISC Zero) can use **20–40 GB** under `target/` during the build. If you see `No space left on device (os error 28)`, free Docker space and raise the disk limit: + +```bash +docker system df +docker builder prune -af # drops build cache (safe before a clean rebuild) +``` + +Docker Desktop → **Settings → Resources → Disk image size** → **64 GB+** (or **Clean / Purge data** if the VM is full), then rebuild. + +**Linker (`__aarch64_cas4_sync` / `__aarch64_swp4_sync`):** On Alpine **arm64**, the prover build uses `-mno-outline-atomics` / `-C target-feature=-outline-atomics` (see Dockerfile `builder-prover` and `.cargo/config.toml`). If you changed those flags, rebuild without cache: `docker buildx build --no-cache-filter builder-prover ...`. + +```bash +docker build -t subs:slim --build-arg ENABLE_PROVER=false . +``` + +```bash +# NVIDIA CUDA prover (Linux hosts with GPU) +docker build -t subs:cuda --build-arg GPU_ACCELERATION=cuda . + +# Apple Metal prover (macOS/arm64 builds) +docker build -t subs:metal --build-arg GPU_ACCELERATION=metal . +``` + +### Run `subs` + +Point at a `spaced` instance reachable from the container (use `host.docker.internal` on Docker Desktop for a node on the host): + +```bash +docker run --rm \ + -p 7777:7777 -p 8888:8888 -p 8080:8080 -p 8081:8081 \ + -v subs-data:/data \ + -e SUBS_SPACED_RPC_URL=http://host.docker.internal:7225 \ + -e SUBS_WALLET=my-wallet \ + -e SUBS_SPACED_RPC_USER=testuser \ + -e SUBS_SPACED_RPC_PASSWORD=secret \ + subs:latest subs +``` + +(`SUBS_PROVER_ENDPOINT` and `SUBS_REGISTRY_ENDPOINT` default to `http://127.0.0.1:8888` and `http://127.0.0.1:8081` in the image.) + +Or mount a `.env` file: + +```bash +docker run --rm -p 7777:7777 \ + -v "$(pwd)/.env:/data/.env:ro" \ + -v subs-data:/data \ + -e SUBS_ENV_FILE=/data/.env \ + subs:latest +``` + +### Run `subs-prover` only + +The default `subs` command already starts subs-prover in the same container. To run the prover alone: + +```bash +docker run --rm -p 8888:8888 \ + -e SUBS_START_PROVER=0 \ + -e SUBS_START_REGISTRY=0 \ + subs:latest subs-prover --server +``` + +### Run `registry-server` only + +The default `subs` command already starts registry-server in the same container. To run registry alone: + +```bash +docker run --rm -p 8081:8081 \ + -e SUBS_START_PROVER=0 \ + -e SUBS_START_REGISTRY=0 \ + subs:latest registry-server +``` + +### Docker Compose + +Starts `subs` with embedded subs-prover (8888) and registry-server (8081) in the same container: + +```bash +cp .env.example .env +# Set SUBS_SPACED_RPC_URL=http://host.docker.internal:7225 and SUBS_WALLET=... +docker compose up --build +``` + +Optional standalone services: + +```bash +docker compose --profile prover-only up --build +docker compose --profile registry-only up --build +``` + +Open http://localhost:7777 for the operator UI. Prover and registry APIs are at http://localhost:8888 and http://localhost:8081. Compose sets `SUBS_PROVER_ENDPOINT=http://127.0.0.1:8888` and `SUBS_REGISTRY_ENDPOINT=http://127.0.0.1:8081` by default. + ## Usage ### 1. Start the prover server diff --git a/SETUP_NEW_SPACE.md b/SETUP_NEW_SPACE.md new file mode 100644 index 0000000..ca536fb --- /dev/null +++ b/SETUP_NEW_SPACE.md @@ -0,0 +1,191 @@ +# Setting Up a New Space — Recommended Sequence + +How to stage, commit, prove, broadcast, publish, and resolve handles in a new space using subs. + +--- + +## The correct per-commit cycle + +Every commitment (genesis or later) follows the same skeleton: + +``` +Stage → Commit Local → [Prove if idx ≥ 1] → Broadcast → Confirmed → Publish → Resolve +``` + +**Publish always comes after broadcast confirms on-chain**, not before. + +--- + +## First commitment (genesis, `idx == 0`) + +Recommended for the **first handle(s)** in a new space: + +1. **Operate** the space +2. **Stage** one or more handles (all unstaged, unparked handles go into one commit) +3. **Commit Local** — no proving required +4. **Broadcast** on-chain +5. **Wait until Confirmed** (on-chain tip root matches your commitment) +6. **Publish** certificates +7. **Resolve** via Query UI / `fabric resolve` + +You need **broadcast + confirm** between commit and publish — not `stage → commit → publish` alone. + +You can put **multiple handles in the first commit** — there is no minimum batch size of 2. `commit_local` takes **all staged, unparked handles at once**. + +**Example (first commitment, no prover needed):** + +```bash +BASE=http://127.0.0.1:7777 +SPACE=%40swifty + +curl -X POST "$BASE/spaces/$SPACE/operate" +# Stage handle(s) via UI or POST /requests + +curl -X POST "$BASE/spaces/$SPACE/commit" \ + -H "Content-Type: application/json" -d '{"dry_run":false}' + +curl -X POST "$BASE/spaces/$SPACE/broadcast" \ + -H "Content-Type: application/json" -d '{"fee_rate": 1.0}' + +# Wait until GET .../commit/status shows Confirmed + +curl -X POST "$BASE/spaces/$SPACE/publish" + +curl -X POST "$BASE/query" \ + -H "Content-Type: application/json" -d '{"handle":"test@swifty"}' +``` + +--- + +## Second commitment (`idx == 1`) + +For the next handle(s): + +1. **Stage** the new handle(s) +2. **Commit Local** — creates a non-initial commit +3. **Prove** (STARK step receipt) — **required before broadcast** +4. **Broadcast** +5. **Wait until Confirmed** +6. **Publish** +7. **Resolve** + +Order: **prove → broadcast → wait for confirmation → publish**, not prove → publish → wait. + +**Example (proving required):** + +```bash +curl -X POST "$BASE/spaces/$SPACE/commit" \ + -H "Content-Type: application/json" -d '{"dry_run":false}' + +curl -X POST "$BASE/spaces/$SPACE/proving/push" +curl -X POST "$BASE/spaces/$SPACE/proving/poll" +# Repeat poll until proving step is Complete + +curl -X POST "$BASE/spaces/$SPACE/broadcast" \ + -H "Content-Type: application/json" -d '{"fee_rate": 1.0}' + +# Wait until Confirmed + +curl -X POST "$BASE/spaces/$SPACE/publish" \ + -H "Content-Type: application/json" -d '{"handles":["newhandle"]}' +``` + +--- + +## Third+ commitments (`idx >= 2`) + +Same as the second commitment, but proving may include a **fold** step in addition to the step proof. Commitment index 2+ requires an **aggregate (fold) receipt**, not just the step receipt. + +--- + +## Batching + +There is **no rule that batches must be ≥ 2**. Batching is an operational choice: + +| Layer | Behavior | +|-------|----------| +| **Commit** | All staged, unparked handles in one local commit | +| **Publish** | Up to **100 handles** per publish request | + +Practical guidance: + +- **Genesis commit**: batch as many handles as you want in the first commit to avoid extra on-chain transactions and skip proving entirely for that commit. +- **Later commits**: batch when it makes sense (e.g. weekly registry sync), knowing each batch needs **prove → broadcast → confirm → publish**. +- **Single-handle commits** are fine — just slower and more expensive on-chain. + +--- + +## Gate before the next local commit + +With RPC connected, `can_commit_local` blocks a new commit until the **previous** one is fully settled: + +1. Previous non-genesis commit has **proving complete** +2. Previous commit is **broadcast** +3. Previous commit has **≥ 150 confirmations** (UI “Finalized” step) + +Steady-state rhythm: + +``` +stage batch → commit → prove → broadcast → wait (150 confs) → publish + ↓ + stage next batch +``` + +**Publish does not require 150 confirmations** — only **Confirmed** is enough. The 150-block wait is for starting the **next** local commit. + +--- + +## Parked handles + +Handles marked **parked** are excluded from the next `commit_local`. Unpark them before committing if you want them included. + +--- + +## Quick reference + +| Commitment | Proving | Before publish | Before next commit | +|------------|---------|----------------|------------------| +| Genesis (`idx == 0`) | Skipped | Broadcast + Confirmed | Broadcast + 150 confs | +| Second (`idx == 1`) | Step receipt | Prove + Broadcast + Confirmed | Prove + Broadcast + 150 confs | +| Third+ (`idx >= 2`) | Step + fold receipts | Prove + Broadcast + Confirmed | Prove + Broadcast + 150 confs | + +--- + +## Common mistakes + +| Wrong | Right | +|-------|-------| +| `stage → commit → publish` | `stage → commit → broadcast → confirm → publish` | +| `prove → publish → wait` | `prove → broadcast → confirm → publish` | +| Wait for batch ≥ 2 before committing | Commit any number of staged handles; batching is optional | +| Publish before on-chain confirm | Wait for Confirmed; early publish causes `signature invalid` errors | + +--- + +## Recommended pattern for a new space + +1. **Genesis**: stage N handles → one commit → broadcast → confirm → publish all N. +2. **Ongoing**: accumulate staged handles → when ready, one commit cycle (prove if not genesis) → broadcast → confirm → publish (batched up to 100). + +--- + +## Optional: block publish until finalized + +Set `SUBS_PUBLISH_REQUIRE_FINALIZED=1` (or `--publish-require-finalized`) to enforce stricter publish gating: + +- **Off (default):** original behavior — publish is available whenever handles are unpublished. +- **On:** publish is blocked in the UI and API until each handle's commitment has **150 on-chain confirmations** (pipeline **Finalized** step). + +When blocked, the space page publish button is disabled with a tooltip explaining why. The API returns `409 Conflict` with the same message. + +--- + +## Related docs + +| Topic | File | +|-------|------| +| Full publish walkthrough | `SUBS_PUBLISH.md` | +| Query / resolve mechanics | `VERITAS_RESOLUTION.md` | +| Fixing publish/resolve failures | `FIX_SIGNATURE_INVALID.md` | +| Pipeline implementation | `core/src/app.rs` (`get_pipeline_status`, `can_commit_local`) | +| Commit logic | `core/src/core.rs` (`commit`, `prepare_zk_input`) | diff --git a/SUBS_API.md b/SUBS_API.md new file mode 100644 index 0000000..afc50b9 --- /dev/null +++ b/SUBS_API.md @@ -0,0 +1,466 @@ +# subs API Reference + +HTTP API reference for the `subs` server (`subs/src/routes/*`). + +- Base URL: `http://127.0.0.1:7777` +- Content type: + - Most endpoints: `application/json` + - Proving endpoints (`/proving/next`, `/proving/fulfill`, `/prover/*`): binary payloads +- Path params: + - `:space` should be URL encoded when needed (example: `@mad` -> `%40mad`) + +--- + +## Status & Spaces + +### GET `/status` +Get status for all loaded spaces. + +### GET `/spaces` +List currently loaded/operated spaces. + +Response: +```json +{ "spaces": ["@mad", "@other"] } +``` + +### GET `/spaces/:space` +Get status of a specific space. Loads/creates the local space first. + +### POST `/spaces/:space/operate` +Check wallet delegation and load/create the space for operation. + +Response: +```json +{ "success": true, "space": "@mad" } +``` + +### GET `/spaces/:space/handles` +List handles with pagination and optional filtering. + +Query params: +- `page` (default `1`) +- `per_page` (default `20`) +- `search` (optional) +- `filter` (optional; values used by UI include `all`, `staged`, `committed`, `parked`, `published`, `unpublished`) + +### GET `/spaces/:space/handles/:handle` +Get a single handle record. + +--- + +## Handle Requests + +### POST `/requests` +Stage one or more handle requests. + +Request: +```json +{ + "requests": [ + { + "handle": "alice@mad", + "script_pubkey": "5120...", + "dev_private_key": null + } + ] +} +``` + +### POST `/requests/generate` +Generate a handle request and optional WIF key. + +If `script_pubkey` is omitted, server generates a keypair and returns `private_key` (WIF). + +Request: +```json +{ "handle": "alice@mad" } +``` + +Response: +```json +{ + "request": { + "handle": "alice@mad", + "script_pubkey": "5120...", + "dev_private_key": "L..." + }, + "private_key": "L..." +} +``` + +### POST `/requests/bulk-generate` +Generate and stage many handles in one call. + +Request: +```json +{ + "space": "@mad", + "count": 100, + "prefix": "h" +} +``` + +Response: +```json +{ "staged": 100 } +``` + +--- + +## Fees, Commits, Pipeline, Publish + +### GET `/fees` +Fetch recommended fee rates from mempool.space. + +Response: +```json +{ + "fastestFee": 8, + "halfHourFee": 5, + "hourFee": 3, + "economyFee": 2, + "minimumFee": 1 +} +``` + +### POST `/spaces/:space/commit` +Commit staged handles locally. + +Request: +```json +{ "dry_run": false } +``` + +Notes: +- `dry_run=true` validates commit readiness and returns 400 if blocked. +- non-initial commits require proving before on-chain broadcast. + +### POST `/spaces/:space/rollback-local` +Rollback the last unbroadcast local commitment. + +Response: +```json +{ "ok": true } +``` + +### POST `/spaces/:space/park` +Park/unpark staged handles by explicit list or bulk search/filter. + +Request: +```json +{ + "handles": ["alice", "bob"], + "parked": true, + "search": null, + "filter": null +} +``` + +Response: +```json +{ "updated": 2 } +``` + +### POST `/spaces/:space/remove` +Remove staged handles by explicit list or bulk search/filter. + +Request: +```json +{ + "handles": ["alice"], + "search": null, + "filter": null +} +``` + +Response: +```json +{ "removed": 1 } +``` + +### POST `/spaces/:space/broadcast` +Broadcast latest local commitment on-chain. + +Request: +```json +{ "fee_rate": 2.0 } +``` + +Response: +```json +{ "txid": "..." } +``` + +### GET `/spaces/:space/commit/status` +Get on-chain commit status. + +Response shape: +```json +{ + "status": "none|pending|confirmed|finalized", + "txid": null, + "block_height": null, + "confirmations": null +} +``` + +### GET `/spaces/:space/pipeline` +Get UI stepper/pipeline state. + +Response includes: +- flattened `PipelineStatus` (steps, counts, current step, message) +- `prover_configured` +- `proving_job_active` + +### POST `/spaces/:space/publish` +Publish certificates in batches (max 100 per request). + +Request (optional body): +```json +{ "handles": ["alice", "bob"] } +``` + +If omitted/empty, server publishes from its unpublished selector. + +Response: +```json +{ + "handles_published": 2, + "remaining": 0 +} +``` + +--- + +## Proving Endpoints + +These endpoints are designed for binary borsh payloads. + +### GET `/spaces/:space/proving/next` +Get next proving request as borsh-serialized `Option`. + +Response content-type: `application/octet-stream` + +### POST `/spaces/:space/proving/fulfill` +Submit a proof receipt in compact binary format. + +Payload format: +- 8 bytes: `commitment_id` (`i64`, little-endian) +- 1 byte: `request_type` (`0` step, `1` fold) +- remaining bytes: borsh-serialized receipt + +Response: +```json +{ "success": true, "message": null } +``` + +### POST `/spaces/:space/proving/push` +Push next proving request to configured external prover. + +Response: +```json +{ + "success": true, + "job_id": "uuid", + "message": "proving request submitted to prover" +} +``` + +### POST `/spaces/:space/proving/poll` +Poll external prover for completion and persist receipt when ready. + +Response: +```json +{ + "success": true, + "status": "pending|processing|complete|failed|null", + "complete": false, + "message": null +} +``` + +### GET `/spaces/:space/proving/estimate` +Forward estimate call to configured prover and return estimate JSON as-is. + +### GET `/spaces/:space/compress` +Get SNARK compression input. + +Response: +```json +{ + "input": { + "receipt": "", + "commitment": { "...": "..." } + } +} +``` + +### POST `/spaces/:space/snark` +Save compressed SNARK receipt. + +Request: +```json +{ "receipt": "" } +``` + +Response: +```json +{ "success": true } +``` + +--- + +## Query & Certificates + +### POST `/query` +Resolve one or more handles via fabric. + +Request: +```json +{ "handle": "alice@mad, bob@mad" } +``` + +Response: array of resolved zones (`badge` + `zone`). + +### GET `/query/message?handle=...` +Export binary `.spacemsg` payload for a handle. + +### GET `/query/anchors` +Export root anchors as pretty JSON attachment. + +### GET `/certs/:handle` +Issue certificate(s) for: +- `@space` -> root cert only +- `name@space` -> root + handle cert + +Response: +```json +{ + "root_cert": "", + "handle_cert": "" +} +``` + +--- + +## RPC Console Proxy + +### GET `/rpc/endpoints` +Return endpoint availability + wallet and chain summary. + +### POST `/rpc/spaced` +Proxy JSON-RPC call to configured spaced RPC endpoint. + +Request: +```json +{ + "method": "walletlistspaces", + "params": ["main"] +} +``` + +### POST `/rpc/bitcoin` +Proxy JSON-RPC call to bitcoind endpoint (test-rig mode only). + +### POST `/rpc/mine` +Mine blocks in test-rig mode. + +Request: +```json +{ "count": 1 } +``` + +--- + +## Runtime Configuration API + +### GET `/config` +Read current persisted endpoints: +- `prover_endpoint` +- `registry_endpoint` + +### POST `/config` +Set/clear endpoint values. + +Request: +```json +{ + "prover_endpoint": "http://127.0.0.1:8888", + "registry_endpoint": "http://127.0.0.1:8081" +} +``` + +Notes: +- pass empty string to clear a value +- omitted fields are unchanged + +### POST `/config/test/prover` +Health-check prover endpoint (`GET /health`). + +### POST `/config/test/registry` +Health-check registry endpoint (`GET /health`, fallback to `/`). + +Request for both: +```json +{ "endpoint": "http://127.0.0.1:8888" } +``` + +Response: +```json +{ "success": true, "error": null } +``` + +--- + +## Registry Integration + +### GET `/registry/status` +Check whether registry endpoint is configured. + +### POST `/registry/sync` +Pull pending handles from configured registry (`/pending`), stage them, then acknowledge via `/ack`. + +Response: +```json +{ + "success": true, + "pulled": 10, + "staged": 8, + "errors": [] +} +``` + +### POST `/registry/notify` +Notify registry webhook of committed handles for a given root. + +Request: +```json +{ + "space": "@mad", + "root": "ab935f..." +} +``` + +Response: +```json +{ + "success": true, + "notified": 4, + "message": null +} +``` + +--- + +## Error Handling + +- Most failures use JSON error responses from `json_error(...)` with HTTP 4xx/5xx. +- Common statuses: + - `400`: invalid input, missing config, invalid payload + - `403`: space not delegated to wallet + - `404`: not found (handle, proving request, etc.) + - `500`: internal/storage/runtime errors + - `502`: upstream prover/RPC/registry errors + - `503`: unavailable dependency (e.g., fee source, rpc endpoint missing) + diff --git a/SUBS_PUBLISH.md b/SUBS_PUBLISH.md new file mode 100644 index 0000000..89373e8 --- /dev/null +++ b/SUBS_PUBLISH.md @@ -0,0 +1,339 @@ +# Publishing a handle with subs + +End-to-end sequence to **create → stage → commit → broadcast → publish → resolve** a handle named **`test@swifty`**. + +This document matches the behavior of the subs operator UI and REST API as implemented in `subs-core` and `subs`. + +--- + +## Names and URLs + +| Concept | Example | Notes | +|--------|---------|--------| +| **Space label** (`SLabel`) | `@swifty` | May also be a `bc1q…` sovereignty label on mainnet. Use the exact label from `spaced`. | +| **Full handle** (`SName`) | `test@swifty` | Subname `test` in space `@swifty`. | +| **Space API path** | `/spaces/%40swifty/...` | URL-encode `@` as `%40`. | +| **Local data** | `$SUBS_DATA_DIR/@swifty/` | SQLite + SpaceDB under the space label. | + +Throughout this doc, **`SPACE=@swifty`** and **`HANDLE=test@swifty`**. + +--- + +## Prerequisites + +Before staging `test@swifty`: + +1. **`spaced`** is running and reachable (`SUBS_SPACED_RPC_URL`, credentials if required). +2. **`subs`** is running with the wallet that **operates** the space: + ```bash + subs --rpc-url http://127.0.0.1:7225 --wallet my-wallet --data-dir ./data --port 7777 + ``` +3. **`subs-prover`** is running and configured in subs (Settings or `SUBS_PROVER_ENDPOINT`): + ```bash + subs-prover --server --server-port 8888 + ``` +4. **Wallet delegation**: the wallet must be allowed to operate `@swifty` on-chain (`wallet_can_operate`). For a new space, the sovereignty owner delegates operation to your wallet first. +5. **Fabric / certrelay**: publish and resolve use the fabric network (`Fabric::new()` default seeds). No extra config is required for normal operation; subs builds a chain proof from `spaced` and broadcasts certificates to relays. + +Open the operator UI: **http://127.0.0.1:7777** + +--- + +## Pipeline overview + +Each handle goes through these phases (see the stepper on the space page): + +``` +Stage → Local commit → [Proving] → Broadcast → Confirmed → Finalized → Publish → Resolve +``` + +| Step | What happens | On-chain? | +|------|----------------|-----------| +| **Stage** | Handle + `script_pubkey` stored locally; not in commitment tree | No | +| **Local commit** | Merkle root computed; commitment row in local DB | No | +| **Proving** | STARK proof for step/fold (required for **2nd+** commitments only) | No | +| **Broadcast** | Commit tx sent via `spaced` wallet | Yes | +| **Confirmed** | Commit tx mined; tip root matches local commitment | Yes | +| **Finalized** | ≥ **150** confirmations on that commit (UI step) | Yes | +| **Publish** | Certificates issued + broadcast to fabric relays | Relays | +| **Resolve** | Query fabric for verified zone | Relays | + +**Important:** Do **not** publish until the commitment is **broadcast and visible on-chain**. Publishing while still only locally committed produces **temporary** certificates signed with the space sovereignty key; relays often reject those with `signature invalid for test@swifty`. Wait until **Broadcast** completes and the chain tip includes your root (pipeline **Confirmed** or later). + +--- + +## Step 1 — Operate the space + +Load/create local state for `@swifty` and verify the wallet can operate it. + +**UI:** Dashboard → select **@swifty** (or add via Operate). + +**API:** +```bash +curl -s -X POST "http://127.0.0.1:7777/spaces/%40swifty/operate" +``` + +**Success:** `{ "success": true, "space": "@swifty" }` +**Failure:** `403` if the wallet is not delegated to operate this space. + +--- + +## Step 2 — Stage `test@swifty` + +Register the handle in the local staging area with a `script_pubkey` (hex-encoded script bytes). + +### Option A — Generate keypair (dev / testing) + +**API:** +```bash +# Returns HandleRequest + WIF private key +curl -s -X POST "http://127.0.0.1:7777/requests/generate" \ + -H "Content-Type: application/json" \ + -d '{"handle":"test@swifty"}' +``` + +Take `request` from the response and stage it: + +```bash +curl -s -X POST "http://127.0.0.1:7777/requests" \ + -H "Content-Type: application/json" \ + -d '{"requests":[{"handle":"test@swifty","script_pubkey":""}]}' +``` + +### Option B — Known script pubkey + +```bash +curl -s -X POST "http://127.0.0.1:7777/requests" \ + -H "Content-Type: application/json" \ + -d '{ + "requests": [{ + "handle": "test@swifty", + "script_pubkey": "5120..." + }] + }' +``` + +**UI:** Use handle generation / staging flows on the space page (or import from registry sync). + +**Verify:** +```bash +curl -s "http://127.0.0.1:7777/spaces/%40swifty/handles?filter=staged" +``` + +Handle should show **staged**, no `commitment_root`, `publish_status` null. + +--- + +## Step 3 — Local commit + +Merge staged handles into the space commitment tree (local only). + +**UI:** **Commit Local** on the space pipeline. + +**API:** +```bash +curl -s -X POST "http://127.0.0.1:7777/spaces/%40swifty/commit" \ + -H "Content-Type: application/json" \ + -d '{"dry_run":false}' +``` + +**Success:** `{ "handles_committed": 1, "is_initial": true|false, ... }` + +- **`is_initial: true`** (first commitment, `idx == 0`): no proving required before broadcast. +- **`is_initial: false`**: you **must** complete proving (step 4) before broadcast. + +**Verify:** Handle shows **committed** with a `commitment_root` and `commitment_idx`. + +--- + +## Step 4 — Proving (non-initial commits only) + +Skip this step for the **first** commitment in a space. + +For the second and later commits, subs creates a STARK proving request that must be fulfilled before on-chain broadcast. + +**UI:** **Prove** (requires prover URL in Settings) → polls until complete. + +**API (typical flow):** +```bash +# Submit job to configured prover +curl -s -X POST "http://127.0.0.1:7777/spaces/%40swifty/proving/push" + +# Poll until done (UI does this automatically) +curl -s -X POST "http://127.0.0.1:7777/spaces/%40swifty/proving/poll" +``` + +Alternative: fetch binary request from `GET .../proving/next`, prove offline, `POST .../proving/fulfill`. + +**Verify:** Pipeline step **Proving** = complete; `GET .../proving/next` returns empty. + +--- + +## Step 5 — Broadcast on-chain + +Submit the commitment root to `spaced` / Bitcoin. + +**UI:** **Broadcast On-Chain** (fee modal). + +**API:** +```bash +curl -s -X POST "http://127.0.0.1:7777/spaces/%40swifty/broadcast" \ + -H "Content-Type: application/json" \ + -d '{"fee_rate": 1.0}' +``` + +**Success:** `{ "txid": "..." }` + +**Verify:** +```bash +curl -s "http://127.0.0.1:7777/spaces/%40swifty/pipeline" +curl -s "http://127.0.0.1:7777/spaces/%40swifty/commit/status" +``` + +Pipeline moves to **Confirmed** (mined) then **Finalized** (≥ 150 confirmations). The UI treats **Finalized** as “ready to publish certificates,” but the critical requirement is that the **on-chain tip root matches** your commitment (Confirmed), not necessarily all 150 blocks—though waiting for **Finalized** is recommended. + +--- + +## Step 6 — Publish certificates + +Issue certificates for unpublished handles and broadcast them to the fabric relay network. + +**UI:** **Publish** bar on the space page (batches up to 100 handles per request). + +**API:** +```bash +# All unpublished handles in the space +curl -s -X POST "http://127.0.0.1:7777/spaces/%40swifty/publish" + +# Single handle +curl -s -X POST "http://127.0.0.1:7777/spaces/%40swifty/publish" \ + -H "Content-Type: application/json" \ + -d '{"handles":["test"]}' +``` + +**What subs does internally:** + +1. Select unpublished handles (respecting on-chain confirmed commitment index). +2. **`issue_certs`**: for each handle, build root cert + leaf cert (`test@swifty`). + - If handle is in the tree at **on-chain tip** → **final** leaf cert (inclusion proof, no Schnorr on leaf). + - If not yet on-chain at tip → **temp** leaf cert (exclusion proof + Schnorr signature from operating wallet). +3. **`build_message`**: `build_chain_proof` RPC against `spaced`. +4. **`fabric.broadcast`**: send message bytes to relays. + +**Success:** `{ "handles_published": 1, "remaining": 0 }` + +**Verify:** Handle `publish_status` becomes `temp` or `final` in DB/UI. + +| `publish_status` | Meaning | +|------------------|---------| +| `null` | Not published | +| `temp` | Temp cert on relays; may need republish after chain advances | +| `final` | Final cert; handle commitment is confirmed on-chain | + +**Common failure:** +```text +Could not broadcast message: relay error (400): rejected: signature invalid for test@swifty +``` +Usually caused by publishing **before broadcast confirms**, wrong operating wallet, or stale temp cert. Fix: wait for on-chain commit, then publish again. + +--- + +## Step 7 — Resolve `test@swifty` + +Query the fabric network for the verified zone (after relays have accepted the publish). + +**UI:** **Query** page → enter `test@swifty` → **Resolve**. + +**API:** +```bash +curl -s -X POST "http://127.0.0.1:7777/query" \ + -H "Content-Type: application/json" \ + -d '{"handle":"test@swifty"}' +``` + +**Success:** JSON array of `ResolvedZone` with `badge` (`orange` / `unverified` / `none`) and zone fields (`script_pubkey`, records, etc.). + +Export binary proof bundle: +```bash +curl -s "http://127.0.0.1:7777/query/message?handle=test%40swifty" -o query.spacemsg +``` + +--- + +## Quick reference — minimal curl sequence + +Assume first commitment (`is_initial: true`), prover not needed, space already delegated: + +```bash +BASE=http://127.0.0.1:7777 +SPACE=%40swifty + +curl -X POST "$BASE/spaces/$SPACE/operate" + +curl -X POST "$BASE/requests/generate" -H "Content-Type: application/json" \ + -d '{"handle":"test@swifty"}' | tee /tmp/gen.json + +# Edit: extract script_pubkey from .request and POST /requests + +curl -X POST "$BASE/spaces/$SPACE/commit" -H "Content-Type: application/json" \ + -d '{"dry_run":false}' + +# Wait until broadcast is appropriate (no pending proving) + +curl -X POST "$BASE/spaces/$SPACE/broadcast" -H "Content-Type: application/json" \ + -d '{"fee_rate": 1.0}' + +# Wait for commit tx to confirm on-chain + +curl -X POST "$BASE/spaces/$SPACE/publish" -H "Content-Type: application/json" \ + -d '{"handles":["test"]}' + +curl -X POST "$BASE/query" -H "Content-Type: application/json" \ + -d '{"handle":"test@swifty"}' +``` + +--- + +## Optional — registry-server + +Registry is **not** required for the publish/resolve flow above. It is a separate queue for pulling handle registrations into staging: + +- `POST /registry/sync` — pull pending handles from registry into staging +- `POST /registry/notify` — notify registry after a commitment is finalized + +Use registry when external parties register handles; otherwise stage via `/requests` directly. + +--- + +## State diagram (handle `test`) + +```text +[ staged ] --commit local--> [ committed locally ] + | + broadcast tx + v + [ on-chain at tip ] + | + publish + v + [ temp or final on relays ] + | + resolve + v + [ zone visible in /query ] +``` + +--- + +## Related code + +| Step | Primary implementation | +|------|------------------------| +| Stage | `Operator::add_requests` → `LocalSpace::add_request` | +| Local commit | `POST /spaces/:space/commit` → `Operator::commit_local` | +| Proving | `POST /spaces/:space/proving/*` | +| Broadcast | `POST /spaces/:space/broadcast` → `Operator::commit` | +| Publish | `POST /spaces/:space/publish` → `Operator::publish_certs` → `submit_certs` | +| Resolve | `POST /query` → `Operator::resolve` | + +Pipeline step logic (150 confirmations, publish readiness): `Operator::get_pipeline_status` in `core/src/app.rs`. diff --git a/VERITAS_RESOLUTION.md b/VERITAS_RESOLUTION.md new file mode 100644 index 0000000..e0bfc31 --- /dev/null +++ b/VERITAS_RESOLUTION.md @@ -0,0 +1,186 @@ +# Veritas Handle Resolution + +How the subs Query UI resolves a handle: the method, process, and trust model. + +The Query UI uses the same **fabric relay resolution** path as the `fabric resolve` CLI — not local subs database state. + +--- + +## UI → API + +1. You enter a handle (e.g. `alice@space`) on `/ui/query` and click **Resolve**. +2. The browser POSTs to subs: + +```javascript +const r = await fetch('/query', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ handle }) +}); +``` + +3. The API handler splits on commas (so `alice@space, bob@space` works) and calls `Operator::resolve`: + +```rust +/// POST /query - Resolve one or more comma-separated handles via the fabric network +pub async fn resolve_handle(...) { + let handles: Vec<&str> = body.handle.split(',').map(|s| s.trim()).filter(|s| !s.is_empty()).collect(); + ... + state.operator.resolve(&handles).await +} +``` + +The per-handle page (`handle.html`) uses the same `POST /query` endpoint for its "Query Fabric" action. + +**Key files:** `subs/templates/query.html`, `subs/src/routes/query.rs`, `core/src/app.rs` + +--- + +## Resolution Method: `fabric-resolver` + `libveritas` + +Subs does **not** look up handles from its local SQLite. It queries the **fabric relay network** and cryptographically verifies the response. + +### Step 1 — Pin Bitcoin Root Anchors (Trust Set) + +Before querying relays, subs fetches **root anchors** from **spaced** via RPC and pins them as the trusted anchor set: + +```rust +pub async fn resolve(&self, handles: &[&str]) -> anyhow::Result> { + let fabric = self.require_fabric()?; + + // Refresh anchors before querying so we have the latest chain state + let anchors = self.require_rpc()?.get_root_anchors().await?; + let sets = AnchorSets::from_anchors(anchors); + _ = fabric.trust_from_set(sets.latest().unwrap())?; + + let rb = fabric.resolve_all(handles).await +``` + +This builds a `Veritas` verifier anchored to the latest Bitcoin block roots from your spaced node. All certificate messages returned by relays are checked against that chain state. + +### Step 2 — Bootstrap Relay Pool + +The `Fabric` client uses default relay seeds (unless overridden via `with_fabric_seeds`, e.g. in test-rig): + +- `https://relay-cosmos.spacesprotocol.org` +- `https://relay-atlas.spacesprotocol.org` + +It discovers/bootstrap peers from those seeds before querying. + +### Step 3 — Nested Name Decomposition (`resolve_all`) + +For handles like `hello.alice@space`, resolution is **iterative**: + +```rust +pub async fn resolve_all(&self, handles: &[&str]) -> Result { + let lookup = libveritas::names::Lookup::new(snames); + ... + let mut batch: Vec = lookup.start(); // first level, e.g. alice@space + while !batch.is_empty() { + let (verified, relay_url) = self.resolve_flat(&refs).await?; + batch = lookup.advance(&verified.zones); // follow aliases to deeper levels + all_zones.extend(verified.zones); + } + lookup.expand_zones(&mut all_zones); // expand canonical → full handle names +``` + +Deep names are broken into batches of 2-label lookups (`subname@space`), resolved level by level using alias mappings from prior zones. + +### Step 4 — Query Relays (`GET /query`) + +For each batch, `resolve_flat` groups handles by space and sends a relay query: + +```rust +// Build GET query params +let q_param = q_parts.join(","); // e.g. "@space,gxxxxxx@space" +... +.get(format!("{url}/query")) +.query(&[("q", &q_param)]); +``` + +Relays return a binary **`.spacemsg`** bundle containing: + +- A **chain proof** (Bitcoin anchor + spaces/nums Merkle proofs) +- **Certificate bundles** per space (root cert + leaf certs) + +Subs tries up to 4 relays (preferring those with freshest hints), falling back on failure. + +### Step 5 — Cryptographic Verification (`libveritas`) + +Each relay response is verified by `Veritas::verify_with_options`: + +```rust +match self.veritas.lock().unwrap().verify_with_options(ctx, msg, options) { + Ok(res) => { ... return Ok((res, url.clone())); } + Err(e) => { last_err = Error::Verify(e); } +``` + +Verification includes: + +1. **Anchor check** — message anchor matches trusted root anchors from spaced +2. **Chain proof check** — spaces/nums proofs tie certificates to the Bitcoin anchor +3. **Per-space bundle verification**: + - **Root cert** for `@space`: may require a **ZK receipt** if commitment index > 0 (non-genesis) + - **Leaf cert** for `handle@space`: inclusion proof (final) or exclusion proof + Schnorr signature (temp) +4. **Name expansion** — aliases resolved to full dotted names + +If verification fails, the relay is marked failed and the next relay is tried. If all fail, subs returns an error to the UI. + +### Step 6 — Badge Assignment + +For each verified zone, subs assigns a UI badge based on sovereignty + trust: + +```rust +let badge = fabric.badge_for(zone.sovereignty, &rb.roots); +ResolvedZone { + badge: match badge { + Badge::Orange => "orange", // sovereign + trusted roots → "Verified" + Badge::Unverified => "unverified", + Badge::None => "none", + }.to_string(), + zone, +} +``` + +| Badge | Meaning in UI | +|-------|----------------| +| **orange** ("Verified") | Handle is **sovereign** and verified against your **pinned trusted** root anchors | +| **unverified** | Resolved against **observed** (newer) roots that differ from pinned trust | +| **none** | Dependent/pending sovereignty, or no applicable trust state | + +--- + +## What the UI Displays + +`renderZones()` shows for each returned zone: + +- Handle, sovereignty badge, verification badge, anchor block height +- Script pubkey +- Commitment details (state root, prev root, block height, receipt hash) if present +- Delegate info if present +- Export links: `.spacemsg` binary and `anchors.json` + +Export endpoints rebuild the same verified message bundle for offline verification (`export_message` uses the same fabric + anchor flow). + +--- + +## Important Distinctions + +| Source | What it resolves | +|--------|------------------| +| **Query UI / `POST /query`** | Published certificates on **fabric relays**, verified against **spaced root anchors** | +| **`GET /spaces/:space/handles/:name`** | **Local subs DB** only (staged/committed status) — anonymous, no fabric | +| **`fabric resolve` CLI** | Same fabric path as Query UI (subs wraps the same `fabric-resolver` library) | + +--- + +## When Resolution Succeeds + +Query UI resolution succeeds only if: + +1. Certificates were **published to relays** (Publish step completed without relay rejection) +2. The **root cert includes required ZK receipt** (for non-genesis commitments) +3. **spaced RPC** is available (for root anchors) +4. At least one **relay** returns a verifiable message + +A common failure mode: the root space (`@space`) resolves from chain anchor alone, but a subhandle (e.g. `gxxxxxx@space`) fails because the published cert chain on relays is incomplete or unverifiable — often due to an incomplete proving → broadcast → finalize → publish pipeline. diff --git a/config-origins/Cargo.toml b/config-origins/Cargo.toml new file mode 100644 index 0000000..46dd8b2 --- /dev/null +++ b/config-origins/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "config-origins" +version = "0.1.0" +edition = "2021" +description = "Startup configuration logging with value origins for subs binaries" +publish = false + +[dependencies] +clap = { workspace = true } +dotenvy = { workspace = true } diff --git a/config-origins/src/lib.rs b/config-origins/src/lib.rs new file mode 100644 index 0000000..b471ce3 --- /dev/null +++ b/config-origins/src/lib.rs @@ -0,0 +1,185 @@ +//! Helpers for loading `.env` files and logging effective configuration with origins. + +use std::collections::{HashMap, HashSet}; +use std::fmt; +use std::path::{Path, PathBuf}; + +use clap::parser::ValueSource; +use clap::ArgMatches; + +/// Where a configuration value came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConfigOrigin { + /// Command-line flag or positional argument. + Param, + /// Process environment (e.g. `export VAR=...`) before `.env` was applied. + Environment, + /// `.env` file (or file pointed to by `*_ENV_FILE`). + DotEnv, + /// Built-in default when nothing else was provided. + Default, +} + +impl fmt::Display for ConfigOrigin { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Param => write!(f, "param"), + Self::Environment => write!(f, "environment"), + Self::DotEnv => write!(f, ".env"), + Self::Default => write!(f, "default"), + } + } +} + +/// Result of loading a dotenv file. +#[derive(Debug, Clone)] +pub struct DotenvLoad { + /// Path loaded, if any. + pub env_file: Option, + /// Variable names whose values were applied from the file (not pre-set in the process env). + pub keys_from_dotenv: HashSet, +} + +/// Snapshot of the process environment before loading dotenv. +type EnvSnapshot = HashMap; + +/// Load variables from a `.env` file before CLI parsing. +/// +/// If `env_file_var` is set in the environment, that path is used; otherwise tries `.env` +/// in the current working directory. Existing process environment variables are not overridden. +pub fn load_dotenv(env_file_var: &str) -> DotenvLoad { + let before = snapshot_env(); + let (env_file, file_keys) = resolve_env_file(env_file_var); + + if let Some(ref path) = env_file { + let _ = dotenvy::from_filename(path); + } else { + let _ = dotenvy::dotenv(); + } + + let mut keys_from_dotenv = HashSet::new(); + for key in file_keys { + if std::env::var(&key).is_ok() && !before.contains_key(&key) { + keys_from_dotenv.insert(key); + } + } + + DotenvLoad { + env_file, + keys_from_dotenv, + } +} + +fn snapshot_env() -> EnvSnapshot { + std::env::vars().collect() +} + +fn resolve_env_file(env_file_var: &str) -> (Option, HashSet) { + if let Ok(path) = std::env::var(env_file_var) { + if !path.is_empty() { + let p = PathBuf::from(&path); + let keys = parse_dotenv_keys(&p).unwrap_or_default(); + return (Some(p), keys); + } + } + + let dot_env = PathBuf::from(".env"); + if dot_env.is_file() { + let keys = parse_dotenv_keys(&dot_env).unwrap_or_default(); + (Some(dot_env), keys) + } else { + (None, HashSet::new()) + } +} + +/// Parse variable names from a dotenv file (ignores comments and blank lines). +fn parse_dotenv_keys(path: &Path) -> std::io::Result> { + let content = std::fs::read_to_string(path)?; + let mut keys = HashSet::new(); + for line in content.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let line = line.strip_prefix("export ").unwrap_or(line).trim(); + if let Some((key, _)) = line.split_once('=') { + let key = key.trim(); + if !key.is_empty() { + keys.insert(key.to_string()); + } + } + } + Ok(keys) +} + +/// Origin for a clap argument that may also use an environment variable. +pub fn origin_from_clap( + matches: &ArgMatches, + field_id: &str, + env_var: Option<&str>, + dotenv: &DotenvLoad, +) -> ConfigOrigin { + match matches.value_source(field_id) { + Some(ValueSource::CommandLine) => ConfigOrigin::Param, + Some(ValueSource::EnvVariable) => { + env_var + .and_then(|v| origin_for_env_var(v, dotenv)) + .unwrap_or(ConfigOrigin::Environment) + } + Some(ValueSource::DefaultValue) => ConfigOrigin::Default, + _ => ConfigOrigin::Default, + } +} + +/// Origin for a setting that is only available via environment (not a CLI flag). +pub fn origin_for_env_var(env_var: &str, dotenv: &DotenvLoad) -> Option { + if std::env::var(env_var).is_err() { + return None; + } + if dotenv.keys_from_dotenv.contains(env_var) { + Some(ConfigOrigin::DotEnv) + } else { + Some(ConfigOrigin::Environment) + } +} + +/// Print a startup configuration section to stdout. +pub fn log_section(component: &str, dotenv: &DotenvLoad) { + println!("{component} configuration:"); + if let Some(path) = &dotenv.env_file { + println!(" (loaded env file: {})", path.display()); + } +} + +/// Print one configuration line. +pub fn log_entry(name: &str, value: impl fmt::Display, origin: ConfigOrigin) { + println!(" {name} = {value} ({origin})"); +} + +/// Print one optional configuration line. +pub fn log_entry_optional( + name: &str, + value: Option, + origin: Option, + secret: bool, +) { + match (value, origin) { + (Some(v), Some(o)) => { + if secret { + log_entry(name, "(set)", o); + } else { + log_entry(name, v, o); + } + } + _ => println!(" {name} = (not set)"), + } +} + +/// Display value for sensitive settings. +pub fn display_secret(value: Option<&str>) -> String { + if value.is_some() && !value.unwrap_or("").is_empty() { + "(set)".to_string() + } else { + "(not set)".to_string() + } +} diff --git a/core/src/app.rs b/core/src/app.rs index a804ba8..01b61b8 100644 --- a/core/src/app.rs +++ b/core/src/app.rs @@ -9,6 +9,7 @@ use std::sync::{Arc, Mutex}; use crate::core::{ CompressInput, HealthWarning, LocalSpace, ProvingRequest, SkippedEntry, SpaceStatus, }; +use crate::storage::Handle; use crate::HandleRequest; use anyhow::anyhow; use bitcoin::hashes::{sha256, Hash as BitcoinHash}; @@ -155,6 +156,9 @@ pub struct PipelineStatus { pub estimate: Option, } +/// Minimum on-chain confirmations before publish is allowed when finalization is required. +pub const PUBLISH_FINALIZATION_CONFIRMATIONS: u32 = 150; + impl LiveSpaceInfo { pub async fn issue_cert( &self, @@ -1688,10 +1692,140 @@ impl Operator { Ok(certs) } + /// When `require_finalized` is true, returns an error message if any handle cannot + /// be published yet (not broadcast, not confirmed, or fewer than 150 confirmations). + pub async fn publish_blocked_reason( + &self, + space: &SLabel, + require_finalized: bool, + handles: &[Handle], + ) -> anyhow::Result> { + if !require_finalized || handles.is_empty() { + return Ok(None); + } + + let rpc = self + .require_rpc() + .map_err(|_| anyhow!("RPC required to verify commitment finalization before publish"))?; + let storage = self.get_local_space(space)?.storage(); + let tip_height = rpc.get_server_info().await?.tip.height; + + for h in handles { + let Some(root_hex) = h.commitment_root.as_deref() else { + return Ok(Some(format!( + "handle {} is not committed; cannot publish", + h.name + ))); + }; + + let Some(commitment) = storage.get_commitment_by_root(root_hex).await? else { + return Ok(Some(format!( + "no commitment found for handle {} (root {})", + h.name, root_hex + ))); + }; + + if commitment.commit_txid.is_none() { + return Ok(Some(format!( + "commitment #{} for handle {} is not broadcast on-chain yet", + commitment.idx, h.name + ))); + } + + let mut expected_root = [0u8; 32]; + hex::decode_to_slice(&commitment.root, &mut expected_root) + .map_err(|e| anyhow!("invalid commitment root for #{}: {}", commitment.idx, e))?; + + let on_chain = rpc.get_commitment(space.clone().into(), None).await?; + let Some(chain_commitment) = on_chain else { + return Ok(Some(format!( + "commitment #{} for handle {} is not confirmed on-chain yet", + commitment.idx, h.name + ))); + }; + + if chain_commitment.state_root != expected_root { + return Ok(Some(format!( + "on-chain tip does not match commitment #{} for handle {}", + commitment.idx, h.name + ))); + } + + let confirmations = tip_height.saturating_sub(chain_commitment.block_height); + if confirmations < PUBLISH_FINALIZATION_CONFIRMATIONS { + let remaining = PUBLISH_FINALIZATION_CONFIRMATIONS - confirmations; + return Ok(Some(format!( + "commitment #{} has {}/{} confirmations ({} more needed before publish)", + commitment.idx, + confirmations, + PUBLISH_FINALIZATION_CONFIRMATIONS, + remaining + ))); + } + } + + Ok(None) + } + + + /// Whether publishing is allowed for the next unpublished batch when finalization is required. + pub async fn publish_gate( + &self, + space: &SLabel, + require_finalized: bool, + limit: usize, + ) -> anyhow::Result<(bool, Option)> { + if !require_finalized { + return Ok((true, None)); + } + + let local_space = self.get_local_space(space)?; + let storage = local_space.storage(); + let confirmed_idx = if let Ok(live) = self.get_live_space(space.clone()).await { + if let Some(tip) = live.tip.as_ref().map(|c| c.state_root) { + storage + .get_commitment_by_root(&hex::encode(tip)) + .await? + .map(|c| c.idx) + } else { + None + } + } else { + None + }; + + let handles = storage + .select_handles(crate::storage::HandleSelector::Unpublished( + confirmed_idx, + Some(limit), + )) + .await?; + if handles.is_empty() { + return Ok((true, None)); + } + + let batch: Vec<_> = handles.into_iter().take(limit).collect(); + match self + .publish_blocked_reason(space, require_finalized, &batch) + .await? + { + None => Ok((true, None)), + Some(reason) => Ok((false, Some(reason))), + } + } + /// Publish certificates for unpublished handles, up to `limit` at a time. /// If `only` is non-empty, only publish those specific handle names. + /// When `require_finalized` is true, publish is rejected until each handle's + /// commitment has 150 on-chain confirmations. /// Returns (published_count, remaining_count). - pub async fn publish_certs(&self, space: &SLabel, limit: usize, only: &[String]) -> anyhow::Result<(usize, usize)> { + pub async fn publish_certs( + &self, + space: &SLabel, + limit: usize, + only: &[String], + require_finalized: bool, + ) -> anyhow::Result<(usize, usize)> { self.require_fabric()?; let local_space = self.get_local_space(space)?; @@ -1752,6 +1886,13 @@ impl Operator { batch.len() }; + if let Some(reason) = self + .publish_blocked_reason(space, require_finalized, &batch) + .await? + { + return Err(anyhow!(reason)); + } + let handle_names: Vec = batch .iter() .map(|h| format!("{}@{}", h.name, space.as_str_unprefixed().unwrap()).parse()) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..9475b3e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,70 @@ +services: + subs: + build: . + image: subs:latest + command: ["subs"] + env_file: + - path: .env + required: false + environment: + SUBS_DATA_DIR: /data + # subs-prover and registry-server start in the same container (see docker/entrypoint.sh). + SUBS_PROVER_ENDPOINT: http://127.0.0.1:8888 + SUBS_REGISTRY_ENDPOINT: http://127.0.0.1:8081 + SUBS_START_PROVER: "1" + SUBS_START_REGISTRY: "1" + SUBS_PROVER_SERVER: "1" + SUBS_PROVER_PORT: "8888" + REGISTRY_SERVER_PORT: "8081" + # Required by registry-server (set real values in .env for compose). + REGISTRY_API_KEY: ${REGISTRY_API_KEY:-change-me-intake} + SUBSD_API_KEY: ${SUBSD_API_KEY:-change-me-subsd} + RUST_LOG: subs=info,subs_prover=info,registry_server=info,tower_http=debug + volumes: + - subs-data:/data + ports: + - "${SUBS_PORT:-7777}:7777" + - "${SUBS_PROVER_PORT:-8888}:8888" + - "${REGISTRY_SERVER_PORT:-8081}:8081" + healthcheck: + # /health stays anonymous even when SUBS_BASIC_AUTH_* is configured. + test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:7777/health"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 60s + + # Optional: run a single component alone (embedded services disabled). + subs-prover: + build: . + image: subs:latest + command: ["subs-prover", "--server"] + environment: + SUBS_START_PROVER: "0" + SUBS_START_REGISTRY: "0" + SUBS_PROVER_SERVER: "1" + SUBS_PROVER_PORT: "8888" + RUST_LOG: subs_prover=info,tower_http=debug + ports: + - "${SUBS_PROVER_PORT:-8888}:8888" + profiles: + - prover-only + + registry-server: + build: . + image: subs:latest + command: ["registry-server"] + environment: + SUBS_START_PROVER: "0" + SUBS_START_REGISTRY: "0" + REGISTRY_SERVER_PORT: "8081" + REGISTRY_API_KEY: ${REGISTRY_API_KEY:-change-me-intake} + SUBSD_API_KEY: ${SUBSD_API_KEY:-change-me-subsd} + RUST_LOG: registry_server=info,tower_http=debug + ports: + - "${REGISTRY_SERVER_PORT:-8081}:8081" + profiles: + - registry-only + +volumes: + subs-data: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 0000000..afb3c8a --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,121 @@ +#!/bin/sh +# Dispatch to subs, subs-prover, or registry-server. +# By default, starting subs also starts co-located services when their binaries exist. +# Usage: +# docker run subs [flags...] +# docker run subs-prover --server +# docker run registry-server --port 8081 + +set -eu + +PROVER_PORT="${SUBS_PROVER_PORT:-8888}" +REGISTRY_PORT="${REGISTRY_SERVER_PORT:-8081}" + +# Apply image defaults from build (only for unset variables). +load_image_defaults() { + if [ ! -f /etc/subs-image.env ]; then + return 0 + fi + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + ''|\#*) continue ;; + esac + key="${line%%=*}" + value="${line#*=}" + eval "if [ -z \"\${$key+x}\" ]; then export $key=\"$value\"; fi" + done < /etc/subs-image.env +} + +load_image_defaults + +if [ -n "${SUBS_PROVER_GPU_ACCELERATION:-}" ]; then + echo "entrypoint: subs-prover GPU acceleration: ${SUBS_PROVER_GPU_ACCELERATION}" +fi + +require_binary() { + if [ ! -x "$1" ]; then + echo "entrypoint: $1 is not available in this image (rebuild with ENABLE_PROVER/ENABLE_REGISTRY enabled)" >&2 + exit 1 + fi +} + +# Start subs-prover in the background (co-located with subs). +start_prover_server() { + if [ "${SUBS_START_PROVER:-1}" = "0" ] || [ ! -x /usr/local/bin/subs-prover ]; then + return 0 + fi + echo "entrypoint: starting subs-prover on 127.0.0.1:${PROVER_PORT}" + SUBS_PROVER_SERVER=1 SUBS_PROVER_PORT="${PROVER_PORT}" \ + /usr/local/bin/subs-prover --server --server-port "${PROVER_PORT}" & +} + +# Start registry-server in the background (co-located with subs). +start_registry_server() { + if [ "${SUBS_START_REGISTRY:-1}" = "0" ] || [ ! -x /usr/local/bin/registry-server ]; then + return 0 + fi + echo "entrypoint: starting registry-server on 127.0.0.1:${REGISTRY_PORT}" + /usr/local/bin/registry-server --port "${REGISTRY_PORT}" & +} + +resolve_component() { + if [ -n "${SUBS_COMPONENT:-}" ]; then + printf '%s' "$SUBS_COMPONENT" + return + fi + + if [ "$#" -gt 0 ]; then + case "$1" in + subs|subs-prover|prover|registry-server|registry) + printf '%s' "$1" + return + ;; + esac + fi + + printf '%s' "subs" +} + +COMPONENT="$(resolve_component)" + +case "$COMPONENT" in + subs) + BIN=/usr/local/bin/subs + ;; + subs-prover|prover) + BIN=/usr/local/bin/subs-prover + COMPONENT=subs-prover + require_binary "$BIN" + ;; + registry-server|registry) + BIN=/usr/local/bin/registry-server + COMPONENT=registry-server + require_binary "$BIN" + ;; + *) + echo "entrypoint: unknown SUBS_COMPONENT '$COMPONENT' (expected subs, subs-prover, or registry-server)" >&2 + exit 1 + ;; +esac + +# If the first argument was the component name, shift it off before exec. +if [ "$#" -gt 0 ]; then + case "$1" in + subs|subs-prover|prover|registry-server|registry) + shift + ;; + esac +fi + +if [ "$COMPONENT" = "subs" ]; then + start_prover_server + start_registry_server + if [ -x /usr/local/bin/subs-prover ] && [ -z "${SUBS_PROVER_ENDPOINT:-}" ]; then + export SUBS_PROVER_ENDPOINT="http://127.0.0.1:${PROVER_PORT}" + fi + if [ -x /usr/local/bin/registry-server ] && [ -z "${SUBS_REGISTRY_ENDPOINT:-}" ]; then + export SUBS_REGISTRY_ENDPOINT="http://127.0.0.1:${REGISTRY_PORT}" + fi +fi + +exec "$BIN" "$@" diff --git a/examples/registry-server/Cargo.toml b/examples/registry-server/Cargo.toml index a4cfa73..aa03e64 100644 --- a/examples/registry-server/Cargo.toml +++ b/examples/registry-server/Cargo.toml @@ -10,6 +10,7 @@ name = "registry-server" path = "src/main.rs" [dependencies] +config-origins = { path = "../../config-origins" } axum = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "signal"] } tower-http = { workspace = true } diff --git a/examples/registry-server/README.md b/examples/registry-server/README.md index b71b2e2..4735650 100644 --- a/examples/registry-server/README.md +++ b/examples/registry-server/README.md @@ -32,10 +32,12 @@ This architecture keeps subsd private (it holds wallet keys) while the registry # Build cargo build --release -p registry-server -# Run — both keys are required, and must differ +# Run — both keys are required, and must differ. +# Loads .env from the current directory if present (or REGISTRY_SERVER_ENV_FILE). REGISTRY_API_KEY=$(openssl rand -hex 32) \ SUBSD_API_KEY=$(openssl rand -hex 32) \ -registry-server --port 8080 +registry-server --port 8081 +# REGISTRY_SERVER_PORT=8081 registry-server ``` The server refuses to start if either key is missing or if the two are equal, @@ -43,7 +45,7 @@ so it can never come up unauthenticated by accident. Then configure subsd to use this registry: 1. Go to Settings in the subsd UI -2. Set Registry Endpoint to `http://localhost:8080` +2. Set Registry Endpoint to `http://localhost:8081` 3. Set Auth Token to your `SUBSD_API_KEY` 4. Click Test — it probes `/health` with the token, so it fails on a bad token, not just an unreachable host @@ -83,7 +85,7 @@ subs' Settings. ### Enqueue a handle (your backend) ```bash -curl -X POST http://localhost:8080/register \ +curl -X POST http://localhost:8081/register \ -H "Authorization: Bearer $REGISTRY_API_KEY" \ -H "Content-Type: application/json" \ -d '{ @@ -95,7 +97,7 @@ curl -X POST http://localhost:8080/register \ ### Check status (user — no auth) ```bash -curl http://localhost:8080/status/alice@example +curl http://localhost:8081/status/alice@example ``` ### Get pending handles (subsd) @@ -104,13 +106,13 @@ curl http://localhost:8080/status/alice@example # subsd asks one space at a time; unscoped returns everything. curl -H "Authorization: Bearer $SUBSD_API_KEY" \ --get --data-urlencode "space=@example" \ - http://localhost:8080/pending + http://localhost:8081/pending ``` ### Acknowledge staged (subsd) ```bash -curl -X POST http://localhost:8080/ack \ +curl -X POST http://localhost:8081/ack \ -H "Authorization: Bearer $SUBSD_API_KEY" \ -H "Content-Type: application/json" \ -d '{"handles": [ diff --git a/examples/registry-server/src/main.rs b/examples/registry-server/src/main.rs index ce8439e..7ced4ea 100644 --- a/examples/registry-server/src/main.rs +++ b/examples/registry-server/src/main.rs @@ -23,7 +23,7 @@ //! # Usage //! //! ```bash -//! REGISTRY_API_KEY=... SUBSD_API_KEY=... registry-server --port 8080 +//! REGISTRY_API_KEY=... SUBSD_API_KEY=... registry-server --port 8081 //! ``` use std::env; @@ -38,7 +38,8 @@ use axum::{ routing::{get, post}, Json, Router, }; -use clap::Parser; +use clap::{CommandFactory, FromArgMatches, Parser}; +use config_origins::{load_dotenv, log_entry, log_section, origin_from_clap}; use serde::{Deserialize, Serialize}; use tokio::sync::RwLock; use tower_http::cors::{Any, CorsLayer}; @@ -52,7 +53,7 @@ use tower_http::trace::TraceLayer; )] struct Cli { /// Server port - #[arg(short, long, default_value = "8080")] + #[arg(short, long, env = "REGISTRY_SERVER_PORT", default_value = "8081")] port: u16, } @@ -85,6 +86,8 @@ enum RegistrationStatus { #[tokio::main] async fn main() -> anyhow::Result<()> { + let dotenv = load_dotenv("REGISTRY_SERVER_ENV_FILE"); + tracing_subscriber::fmt() .with_env_filter( tracing_subscriber::EnvFilter::try_from_default_env() @@ -92,7 +95,19 @@ async fn main() -> anyhow::Result<()> { ) .init(); - let cli = Cli::parse(); + let matches = Cli::command().get_matches(); + let cli = Cli::from_arg_matches(&matches).unwrap_or_else(|e| e.exit()); + + log_section("registry-server", &dotenv); + log_entry( + "port", + cli.port, + origin_from_clap(&matches, "port", Some("REGISTRY_SERVER_PORT"), &dotenv), + ); + println!( + " server_url = http://127.0.0.1:{} (derived from port)", + cli.port + ); let registry_api_key = require_env("REGISTRY_API_KEY")?; let subsd_api_key = require_env("SUBSD_API_KEY")?; diff --git a/prover/Cargo.toml b/prover/Cargo.toml index 1f06717..c9e53d6 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -15,10 +15,12 @@ path = "src/main.rs" risc0-zkvm = { workspace = true, features = ["prove"] } libveritas = { workspace = true } libveritas_zk = { workspace = true } +config-origins = { path = "../config-origins" } subs-types = { workspace = true } spacedb = { workspace = true } clap = { workspace = true } +dotenvy = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } anyhow = { workspace = true } diff --git a/prover/src/env.rs b/prover/src/env.rs new file mode 100644 index 0000000..0e733ab --- /dev/null +++ b/prover/src/env.rs @@ -0,0 +1,96 @@ +//! Environment variable and `.env` file loading. + +use clap::ArgMatches; +use config_origins::{ + self as origins, origin_for_env_var, origin_from_clap, DotenvLoad, +}; + +pub use config_origins::load_dotenv; + +/// Log effective `subs-prover` configuration for server mode. +pub fn log_server_startup( + matches: &ArgMatches, + dotenv: &DotenvLoad, + server: bool, + port: u16, + calibrate: bool, +) { + origins::log_section("subs-prover", dotenv); + origins::log_entry( + "server", + server, + origin_from_clap(matches, "server", Some("SUBS_PROVER_SERVER"), dotenv), + ); + origins::log_entry( + "server_port", + port, + origin_from_clap(matches, "server_port", Some("SUBS_PROVER_PORT"), dotenv), + ); + + let calibrate_origin = if matches.get_flag("calibrate") { + origin_from_clap(matches, "calibrate", Some("PROVER_CALIBRATE"), dotenv) + } else if std::env::var("PROVER_CALIBRATE").is_ok_and(|v| !v.is_empty() && v != "0") { + origin_for_env_var("PROVER_CALIBRATE", dotenv).unwrap_or(origins::ConfigOrigin::Environment) + } else { + origins::ConfigOrigin::Default + }; + origins::log_entry("calibrate", calibrate, calibrate_origin); + + let auth_token = std::env::var("PROVER_AUTH_TOKEN").ok().filter(|s| !s.is_empty()); + let auth_origin = origin_for_env_var("PROVER_AUTH_TOKEN", dotenv); + origins::log_entry_optional( + "prover_auth_token", + auth_token.as_deref().map(|_| "***"), + auth_origin, + false, + ); + + println!(" server_url = http://127.0.0.1:{} (derived from server_port)", port); +} + +/// Log configuration for a prove/compress subcommand. +pub fn log_subcommand_startup( + sub: &ArgMatches, + dotenv: &DotenvLoad, + sub_name: &str, + input: Option<&std::path::Path>, + output: Option<&std::path::Path>, +) { + origins::log_section("subs-prover", dotenv); + println!(" command = {sub_name} (param)"); + + log_io_path(sub, "input", "SUBS_PROVER_INPUT", input, dotenv); + log_io_path(sub, "output", "SUBS_PROVER_OUTPUT", output, dotenv); +} + +/// Log configuration for the bench subcommand. +pub fn log_bench_startup(dotenv: &DotenvLoad, sub: &ArgMatches, existing: usize, insert: usize) { + origins::log_section("subs-prover", dotenv); + println!(" command = bench (param)"); + origins::log_entry( + "bench_existing", + existing, + origin_from_clap(sub, "existing", Some("SUBS_PROVER_BENCH_EXISTING"), dotenv), + ); + origins::log_entry( + "bench_insert", + insert, + origin_from_clap(sub, "insert", Some("SUBS_PROVER_BENCH_INSERT"), dotenv), + ); +} + +fn log_io_path( + sub: &ArgMatches, + field_id: &str, + env_var: &str, + value: Option<&std::path::Path>, + dotenv: &DotenvLoad, +) { + let display = value.map(|p| p.display().to_string()); + let origin = match sub.value_source(field_id) { + Some(_) => Some(origin_from_clap(sub, field_id, Some(env_var), dotenv)), + None if display.is_some() => origin_for_env_var(env_var, dotenv), + None => None, + }; + origins::log_entry_optional(field_id, display.as_deref(), origin, false); +} diff --git a/prover/src/lib.rs b/prover/src/lib.rs index 4ee5d8f..cdf9180 100644 --- a/prover/src/lib.rs +++ b/prover/src/lib.rs @@ -2,6 +2,7 @@ //! //! Provides the `Prover` struct for generating STARK proofs and SNARK compression. +pub mod env; pub mod server; use std::time::Instant; diff --git a/prover/src/main.rs b/prover/src/main.rs index 5c320e1..36787c6 100644 --- a/prover/src/main.rs +++ b/prover/src/main.rs @@ -21,7 +21,7 @@ use std::io::{self, Read, Write}; use std::path::PathBuf; use anyhow::Result; -use clap::{Parser, Subcommand}; +use clap::{CommandFactory, FromArgMatches, Parser, Subcommand}; use subs_prover::Prover; use subs_types::{CompressInput, ProvingRequest}; @@ -33,11 +33,11 @@ use subs_types::{CompressInput, ProvingRequest}; )] struct Cli { /// Run as an HTTP server that accepts proving requests - #[arg(long)] + #[arg(long, env = "SUBS_PROVER_SERVER")] server: bool, /// Server port (for --server mode) - #[arg(long, default_value = "8888")] + #[arg(long, env = "SUBS_PROVER_PORT", default_value = "8888")] server_port: u16, /// Run startup calibration (for --server mode). @@ -59,57 +59,87 @@ enum Commands { /// Prove a ProvingRequest (Step or Fold) Prove { /// Input file (JSON ProvingRequest). If not provided, reads from stdin. - #[arg(short, long)] + #[arg(short, long, env = "SUBS_PROVER_INPUT")] input: Option, /// Output file for receipt. If not provided, writes to stdout. - #[arg(short, long)] + #[arg(short, long, env = "SUBS_PROVER_OUTPUT")] output: Option, }, /// Compress a STARK proof to SNARK (Groth16) Compress { /// Input file (JSON CompressInput). If not provided, reads from stdin. - #[arg(short, long)] + #[arg(short, long, env = "SUBS_PROVER_INPUT")] input: Option, /// Output file for receipt. If not provided, writes to stdout. - #[arg(short, long)] + #[arg(short, long, env = "SUBS_PROVER_OUTPUT")] output: Option, }, /// Benchmark: estimate proving cost for inserting handles into a tree Bench { /// Number of existing handles in the tree - #[arg(long, default_value = "10000")] + #[arg(long, env = "SUBS_PROVER_BENCH_EXISTING", default_value = "10000")] existing: usize, /// Number of new handles to insert - #[arg(long, default_value = "100")] + #[arg(long, env = "SUBS_PROVER_BENCH_INSERT", default_value = "100")] insert: usize, }, } #[tokio::main] async fn main() -> Result<()> { - let cli = Cli::parse(); + let dotenv = subs_prover::env::load_dotenv("SUBS_PROVER_ENV_FILE"); + let matches = Cli::command().get_matches(); + let cli = Cli::from_arg_matches(&matches).unwrap_or_else(|e| e.exit()); if cli.server { let calibrate = cli.calibrate || std::env::var("PROVER_CALIBRATE").is_ok_and(|v| !v.is_empty() && v != "0"); + subs_prover::env::log_server_startup( + &matches, + &dotenv, + cli.server, + cli.server_port, + calibrate, + ); subs_prover::server::run_server(cli.server_port, calibrate).await?; return Ok(()); } match cli.cmd { Some(Commands::Prove { input, output }) => { + if let Some((_, sub)) = matches.subcommand() { + subs_prover::env::log_subcommand_startup( + sub, + &dotenv, + "prove", + input.as_deref(), + output.as_deref(), + ); + } let input_data = read_input(input)?; let request: ProvingRequest = serde_json::from_slice(&input_data)?; let receipt = prove(&request)?; write_output(output, &receipt)?; } Some(Commands::Compress { input, output }) => { + if let Some((_, sub)) = matches.subcommand() { + subs_prover::env::log_subcommand_startup( + sub, + &dotenv, + "compress", + input.as_deref(), + output.as_deref(), + ); + } let input_data = read_input(input)?; let compress_input: CompressInput = serde_json::from_slice(&input_data)?; let receipt = compress(&compress_input)?; write_output(output, &receipt)?; } Some(Commands::Bench { existing, insert }) => { + if let Some((_, sub)) = matches.subcommand() { + subs_prover::env::log_bench_startup(&dotenv, sub, existing, insert); + } run_bench(existing, insert)?; } None => { diff --git a/setup_subs_env.sh b/setup_subs_env.sh new file mode 100644 index 0000000..441b48c --- /dev/null +++ b/setup_subs_env.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Source this file with: source setup-subsd-env.sh + +export PS1='subs:\w$ ' +export PATH="$HOME/.cargo/bin:$PATH" +export SUBS_PORT=7244 +export SUBS_DATA_DIR=./datamad +export SUBS_WALLET=mad +export SUBS_SPACED_RPC_URL=http://127.0.0.1:7225 +export SUBS_SPACED_RPC_USER=testuser +export SUBS_SPACED_RPC_PASSWORD=SomeRisk84 +export RUST_LOG=subs=info,error + +alias subs='target/release/subs ' +alias prover='target/release/subs-prover ' +echo "cargo build --release --features metal --bin subs" +echo "cargo run --release --bin subs" +echo "prover --server --server-port 8888" diff --git a/subs/Cargo.toml b/subs/Cargo.toml index 3c1f312..27e9fc1 100644 --- a/subs/Cargo.toml +++ b/subs/Cargo.toml @@ -10,6 +10,7 @@ name = "subs" path = "src/main.rs" [dependencies] +config-origins = { path = "../config-origins" } subs-core = { workspace = true } subs-types = { workspace = true } @@ -20,6 +21,7 @@ tower-http = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } clap = { workspace = true } +dotenvy = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } anyhow = { workspace = true } diff --git a/subs/src/background.rs b/subs/src/background.rs index 2c01fe7..fc5e3b4 100644 --- a/subs/src/background.rs +++ b/subs/src/background.rs @@ -104,7 +104,7 @@ async fn registry_loop(state: AppState) { match state .operator - .publish_certs(space, PUBLISH_BATCH_SIZE, &[]) + .publish_certs(space, PUBLISH_BATCH_SIZE, &[], state.publish_require_finalized) .await { Ok((0, 0)) => {} diff --git a/subs/src/env.rs b/subs/src/env.rs new file mode 100644 index 0000000..b8f6d56 --- /dev/null +++ b/subs/src/env.rs @@ -0,0 +1,270 @@ +//! Environment variable and `.env` file loading. + +use std::path::Path; + +use anyhow::Result; +use clap::ArgMatches; +use config_origins::{ + self as origins, display_secret, origin_for_env_var, origin_from_clap, DotenvLoad, +}; + +use crate::config::ConfigStore; + +pub use config_origins::load_dotenv; + +const PUBLISH_REQUIRE_FINALIZED_ENV: &str = "SUBS_PUBLISH_REQUIRE_FINALIZED"; + +/// Parse a boolean env var. Unset, empty, and unrecognized values default to `false`. +/// Truthy: `1`, `true`, `yes`, `on` (case-insensitive). +pub fn env_bool_default_false(var: &str) -> bool { + match std::env::var(var) { + Ok(v) => matches!( + v.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ), + Err(_) => false, + } +} + +/// Resolve publish finalization gating. +/// +/// Precedence: CLI `--publish-require-finalized` > `SUBS_PUBLISH_REQUIRE_FINALIZED` > `false`. +pub fn resolve_publish_require_finalized(matches: &ArgMatches, cli_flag: bool) -> bool { + if matches!( + matches.value_source("publish_require_finalized"), + Some(clap::parser::ValueSource::CommandLine) + ) { + return cli_flag; + } + env_bool_default_false(PUBLISH_REQUIRE_FINALIZED_ENV) +} + +/// Parsed configuration values used for startup logging. +pub struct StartupValues<'a> { + pub port: u16, + pub data_dir: &'a Path, + pub wallet: Option<&'a str>, + pub rpc_url: Option<&'a str>, + pub rpc_user: Option<&'a str>, + pub rpc_password: Option<&'a str>, + pub rpc_cookie: Option<&'a Path>, + pub basic_auth_user: Option<&'a str>, + pub basic_auth_password: Option<&'a str>, + pub publish_require_finalized: bool, + #[cfg(feature = "test-rig")] + pub test_rig: bool, + #[cfg(feature = "test-rig")] + pub test_rig_dir: &'a Path, +} + +/// Log effective `subs` configuration and each value's origin. +pub fn log_startup(matches: &ArgMatches, dotenv: &DotenvLoad, cfg: StartupValues<'_>) { + origins::log_section("subs", dotenv); + + origins::log_entry( + "port", + cfg.port, + origin_from_clap(matches, "port", Some("SUBS_PORT"), dotenv), + ); + origins::log_entry( + "data_dir", + cfg.data_dir.display(), + origin_from_clap(matches, "data_dir", Some("SUBS_DATA_DIR"), dotenv), + ); + + log_field( + matches, + "wallet", + "SUBS_WALLET", + cfg.wallet, + dotenv, + false, + ); + log_field( + matches, + "rpc_url", + "SUBS_SPACED_RPC_URL", + cfg.rpc_url, + dotenv, + false, + ); + log_field( + matches, + "rpc_user", + "SUBS_SPACED_RPC_USER", + cfg.rpc_user, + dotenv, + false, + ); + log_field( + matches, + "rpc_password", + "SUBS_SPACED_RPC_PASSWORD", + cfg.rpc_password, + dotenv, + true, + ); + let rpc_cookie = cfg + .rpc_cookie + .map(|p| p.display().to_string()); + log_field( + matches, + "rpc_cookie", + "SUBS_SPACED_RPC_COOKIE", + rpc_cookie.as_deref(), + dotenv, + false, + ); + + log_field( + matches, + "basic_auth_user", + "SUBS_BASIC_AUTH_USER", + cfg.basic_auth_user, + dotenv, + false, + ); + log_field( + matches, + "basic_auth_password", + "SUBS_BASIC_AUTH_PASSWORD", + cfg.basic_auth_password, + dotenv, + true, + ); + + log_env_only("prover_endpoint", "SUBS_PROVER_ENDPOINT", dotenv, false); + log_env_only("registry_endpoint", "SUBS_REGISTRY_ENDPOINT", dotenv, false); + + let publish_origin = if matches!( + matches.value_source("publish_require_finalized"), + Some(clap::parser::ValueSource::CommandLine) + ) { + Some(origin_from_clap( + matches, + "publish_require_finalized", + Some(PUBLISH_REQUIRE_FINALIZED_ENV), + dotenv, + )) + } else if std::env::var(PUBLISH_REQUIRE_FINALIZED_ENV).is_ok() { + origin_for_env_var(PUBLISH_REQUIRE_FINALIZED_ENV, dotenv) + } else { + None + }; + origins::log_entry_optional( + "publish_require_finalized", + Some(if cfg.publish_require_finalized { + "true" + } else { + "false" + }), + publish_origin, + false, + ); + + #[cfg(feature = "test-rig")] + { + origins::log_entry( + "test_rig", + cfg.test_rig, + origin_from_clap(matches, "test_rig", Some("SUBS_TEST_RIG"), dotenv), + ); + origins::log_entry( + "test_rig_dir", + cfg.test_rig_dir.display(), + origin_from_clap(matches, "test_rig_dir", Some("SUBS_TEST_RIG_DIR"), dotenv), + ); + } + + println!( + " server_url = http://127.0.0.1:{} (derived from port)", + cfg.port + ); +} + +fn log_field( + matches: &ArgMatches, + field_id: &str, + env_var: &str, + value: Option<&str>, + dotenv: &DotenvLoad, + secret: bool, +) { + let origin = match matches.value_source(field_id) { + Some(_) => Some(origin_from_clap(matches, field_id, Some(env_var), dotenv)), + None if value.is_some() && origin_for_env_var(env_var, dotenv).is_some() => { + origin_for_env_var(env_var, dotenv) + } + None => None, + }; + + if secret { + let display = display_secret(value); + if let Some(o) = origin { + origins::log_entry(field_id, display, o); + } else { + println!(" {field_id} = {display}"); + } + } else { + origins::log_entry_optional(field_id, value, origin, false); + } +} + +fn log_env_only(name: &str, env_var: &str, dotenv: &DotenvLoad, secret: bool) { + let value = std::env::var(env_var).ok(); + let origin = origin_for_env_var(env_var, dotenv); + if secret { + origins::log_entry_optional(name, value.as_deref().map(|_| "(set)"), origin, true); + } else { + origins::log_entry_optional(name, value.as_deref(), origin, false); + } +} + +/// Apply optional runtime settings from the environment into `config.db`. +pub fn apply_runtime_config_from_env(config: &ConfigStore) -> Result<()> { + if let Ok(url) = std::env::var("SUBS_PROVER_ENDPOINT") { + let url = url.trim(); + if !url.is_empty() { + config.set_prover_endpoint(url)?; + } + } + if let Ok(url) = std::env::var("SUBS_REGISTRY_ENDPOINT") { + let url = url.trim(); + if !url.is_empty() { + config.set_registry_endpoint(url)?; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn env_bool_default_false_unset_is_false() { + let key = "SUBS_PUBLISH_REQUIRE_FINALIZED_TEST_UNSET"; + std::env::remove_var(key); + assert!(!env_bool_default_false(key)); + } + + #[test] + fn env_bool_default_false_truthy_values() { + let key = "SUBS_PUBLISH_REQUIRE_FINALIZED_TEST_TRUTHY"; + for v in ["1", "true", "TRUE", " yes ", "on"] { + std::env::set_var(key, v); + assert!(env_bool_default_false(key), "expected true for {v:?}"); + } + std::env::remove_var(key); + } + + #[test] + fn env_bool_default_false_falsey_values() { + let key = "SUBS_PUBLISH_REQUIRE_FINALIZED_TEST_FALSEY"; + for v in ["", "0", "false", "no", "off", "maybe"] { + std::env::set_var(key, v); + assert!(!env_bool_default_false(key), "expected false for {v:?}"); + } + std::env::remove_var(key); + } +} diff --git a/subs/src/main.rs b/subs/src/main.rs index f8b8f84..340288b 100644 --- a/subs/src/main.rs +++ b/subs/src/main.rs @@ -14,6 +14,7 @@ mod background; mod config; +mod env; mod logs; mod routes; mod state; @@ -25,7 +26,8 @@ use std::net::SocketAddr; use std::path::PathBuf; use anyhow::Result; -use clap::Parser; +use axum::middleware; +use clap::{CommandFactory, FromArgMatches, Parser}; use subs_core::Operator; use tower_http::cors::{Any, CorsLayer}; use tower_http::trace::TraceLayer; @@ -42,79 +44,117 @@ use crate::state::AppState; )] struct Cli { /// Server port - #[arg(short, long, default_value = "7777")] + #[arg(short, long, env = "SUBS_PORT", default_value = "7777")] port: u16, /// Data directory for spaces - #[arg(short, long, default_value = "./data")] + #[arg(short, long, env = "SUBS_DATA_DIR", default_value = "./data")] data_dir: PathBuf, /// Wallet name for signing operations (not required with --test-rig) - #[arg(short, long, required_unless_present = "test_rig")] + #[arg(short, long, env = "SUBS_WALLET", required_unless_present = "test_rig")] wallet: Option, /// Spaces RPC URL (not required with --test-rig) - #[arg(short, long, required_unless_present = "test_rig")] + #[arg(short, long, env = "SUBS_SPACED_RPC_URL", required_unless_present = "test_rig")] rpc_url: Option, /// RPC username (optional) - #[arg(long)] + #[arg(long, env = "SUBS_SPACED_RPC_USER")] rpc_user: Option, /// RPC password (optional) - #[arg(long)] + #[arg(long, env = "SUBS_SPACED_RPC_PASSWORD")] rpc_password: Option, /// RPC cookie file path (optional) - #[arg(long)] + #[arg(long, env = "SUBS_SPACED_RPC_COOKIE")] rpc_cookie: Option, + /// HTTP Basic auth username for the UI/API (enables auth when set with a password) + #[arg(long, env = "SUBS_BASIC_AUTH_USER")] + basic_auth_user: Option, + + /// HTTP Basic auth password for the UI/API (enables auth when set with a username) + #[arg(long, env = "SUBS_BASIC_AUTH_PASSWORD")] + basic_auth_password: Option, + + /// Block certificate publish until commitments are finalized (150 confirmations). + /// When unset, defaults to false. Set env `SUBS_PUBLISH_REQUIRE_FINALIZED=1` to enable. + #[arg(long, default_value_t = false)] + publish_require_finalized: bool, + /// Enable test rig mode (starts bitcoind + spaced automatically) #[cfg(feature = "test-rig")] - #[arg(long)] + #[arg(long, env = "SUBS_TEST_RIG")] test_rig: bool, /// Directory for test rig data (persistent across restarts) #[cfg(feature = "test-rig")] - #[arg(long, default_value = "./testrig-data")] + #[arg(long, env = "SUBS_TEST_RIG_DIR", default_value = "./testrig-data")] test_rig_dir: PathBuf, } #[tokio::main] async fn main() -> Result<()> { + let dotenv = env::load_dotenv("SUBS_ENV_FILE"); + // Initialize tracing. The capture layer feeds the Logs page and is // installed here so nothing emitted after startup is missed. tracing_subscriber::registry() .with( tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| "subsd=info,tower_http=debug".into()), + .unwrap_or_else(|_| "subs=info,tower_http=debug".into()), ) .with(tracing_subscriber::fmt::layer()) .with(logs::capture_layer()) .init(); - let cli = Cli::parse(); + let matches = Cli::command().get_matches(); + let cli = Cli::from_arg_matches(&matches).unwrap_or_else(|e| e.exit()); + let publish_require_finalized = + env::resolve_publish_require_finalized(&matches, cli.publish_require_finalized); + env::log_startup( + &matches, + &dotenv, + env::StartupValues { + port: cli.port, + data_dir: &cli.data_dir, + wallet: cli.wallet.as_deref(), + rpc_url: cli.rpc_url.as_deref(), + rpc_user: cli.rpc_user.as_deref(), + rpc_password: cli.rpc_password.as_deref(), + rpc_cookie: cli.rpc_cookie.as_deref(), + basic_auth_user: cli.basic_auth_user.as_deref(), + basic_auth_password: cli.basic_auth_password.as_deref(), + publish_require_finalized, + #[cfg(feature = "test-rig")] + test_rig: cli.test_rig, + #[cfg(feature = "test-rig")] + test_rig_dir: &cli.test_rig_dir, + }, + ); #[cfg(feature = "test-rig")] { if cli.test_rig { - let mut handle = run_with_test_rig(cli).await?; + let mut handle = run_with_test_rig(cli, publish_require_finalized).await?; // Gracefully stop bitcoind so it flushes blocks to disk if let Err(e) = handle.stop().await { tracing::warn!("Failed to stop bitcoind cleanly: {}", e); } } else { - run_normal(cli).await?; + run_normal(cli, publish_require_finalized).await?; } } #[cfg(not(feature = "test-rig"))] - run_normal(cli).await?; + run_normal(cli, publish_require_finalized).await?; Ok(()) } -async fn run_normal(cli: Cli) -> Result<()> { +async fn run_normal(cli: Cli, publish_require_finalized: bool) -> Result<()> { use spaces_client::rpc::RpcClient; let rpc_url = cli.rpc_url.as_ref().expect("rpc_url required"); @@ -144,6 +184,7 @@ async fn run_normal(cli: Cli) -> Result<()> { // Create config store let config_path = cli.data_dir.join("config.db"); let config = ConfigStore::open(&config_path)?; + env::apply_runtime_config_from_env(&config)?; // Create operator let operator = Operator::new(cli.data_dir, wallet, rpc) @@ -152,12 +193,30 @@ async fn run_normal(cli: Cli) -> Result<()> { // Load all existing spaces from disk operator.load_all_spaces().await?; + // Resolve optional HTTP Basic auth for the UI/API + let basic_auth = resolve_basic_auth( + cli.basic_auth_user.as_deref(), + cli.basic_auth_password.as_deref(), + ); + // Build app state and run server - run_server(operator, config, cli.port, Some(rpc_url.clone()), None).await + run_server( + operator, + config, + cli.port, + Some(rpc_url.clone()), + cli.rpc_user.clone(), + cli.rpc_password.clone(), + cli.rpc_cookie.clone(), + basic_auth, + publish_require_finalized, + None, + ) + .await } #[cfg(feature = "test-rig")] -async fn run_with_test_rig(cli: Cli) -> Result { +async fn run_with_test_rig(cli: Cli, publish_require_finalized: bool) -> Result { use std::sync::Arc; use crate::testrig::TestRigHandle; @@ -196,6 +255,7 @@ async fn run_with_test_rig(cli: Cli) -> Result { // Create config store let config_path = cli.data_dir.join("config.db"); let config = ConfigStore::open(&config_path)?; + env::apply_runtime_config_from_env(&config)?; // Use default wallet from test rig let wallet = "wallet_99"; @@ -208,10 +268,16 @@ async fn run_with_test_rig(cli: Cli) -> Result { // Load all existing spaces from disk operator.load_all_spaces().await?; + // Resolve optional HTTP Basic auth for the UI/API + let basic_auth = resolve_basic_auth( + cli.basic_auth_user.as_deref(), + cli.basic_auth_password.as_deref(), + ); + // Run server (this blocks until shutdown) let spaced_url = handle.spaced_rpc_url().to_string(); let bitcoin_url = handle.bitcoin_rpc_url().to_string(); - run_server_with_testrig(operator, config, cli.port, spaced_url, bitcoin_url, certrelay_url, handle.clone()).await?; + run_server_with_testrig(operator, config, cli.port, spaced_url, bitcoin_url, certrelay_url, basic_auth, publish_require_finalized, handle.clone()).await?; // Background tasks (proving loop) hold AppState clones with Arc refs. // On shutdown just leak them; the process is exiting anyway. @@ -224,19 +290,36 @@ async fn run_with_test_rig(cli: Cli) -> Result { } } +#[allow(clippy::too_many_arguments)] async fn run_server( operator: Operator, config: ConfigStore, port: u16, spaced_rpc_url: Option, + spaced_rpc_user: Option, + spaced_rpc_password: Option, + spaced_rpc_cookie: Option, + basic_auth: Option<(String, String)>, + publish_require_finalized: bool, bitcoin_rpc_url: Option, ) -> Result<()> { // Build app state - let state = AppState::with_rpc_urls(operator, config, spaced_rpc_url, bitcoin_rpc_url); + let state = AppState::with_rpc_urls( + operator, + config, + spaced_rpc_url, + spaced_rpc_user, + spaced_rpc_password, + spaced_rpc_cookie, + basic_auth, + publish_require_finalized, + bitcoin_rpc_url, + ); run_server_inner(state, port).await } #[cfg(feature = "test-rig")] +#[allow(clippy::too_many_arguments)] async fn run_server_with_testrig( operator: Operator, config: ConfigStore, @@ -244,10 +327,24 @@ async fn run_server_with_testrig( spaced_rpc_url: String, bitcoin_rpc_url: String, certrelay_url: String, + basic_auth: Option<(String, String)>, + publish_require_finalized: bool, test_rig: std::sync::Arc, ) -> Result<()> { // Build app state with test rig - let state = AppState::with_test_rig(operator, config, Some(spaced_rpc_url), Some(bitcoin_rpc_url), Some(certrelay_url), test_rig); + let state = AppState::with_test_rig( + operator, + config, + Some(spaced_rpc_url), + Some("user".to_string()), + Some("pass".to_string()), + None, + basic_auth, + publish_require_finalized, + Some(bitcoin_rpc_url), + Some(certrelay_url), + test_rig, + ); run_server_inner(state, port).await } @@ -258,8 +355,17 @@ async fn run_server_inner(state: AppState, port: u16) -> Result<()> { // Start background registry loop (idles unless auto-sync is enabled) background::spawn_registry_loop(state.clone()); - // Build router + // Build router. + // + // Layer ordering note: the last `.layer(...)` added runs first on the request. + // The auth layer is added before CORS/trace in the builder chain so that on the + // request path CORS runs first (handling preflight) and auth runs just before the + // handlers. The auth middleware also explicitly allows OPTIONS and /health through. let app = routes::router() + .layer(middleware::from_fn_with_state( + state.clone(), + routes::auth::require_basic_auth, + )) .layer(TraceLayer::new_for_http()) .layer( CorsLayer::new() @@ -272,6 +378,7 @@ async fn run_server_inner(state: AppState, port: u16) -> Result<()> { // Start server let addr = SocketAddr::from(([0, 0, 0, 0], port)); tracing::info!("Starting server on http://{}", addr); + tracing::info!("Server URL: http://127.0.0.1:{}", port); let listener = tokio::net::TcpListener::bind(addr).await?; axum::serve(listener, app) @@ -281,6 +388,30 @@ async fn run_server_inner(state: AppState, port: u16) -> Result<()> { Ok(()) } +/// Resolve the HTTP Basic auth credentials from CLI/env. +/// +/// Auth is only enabled when both username and password are provided. If only one is +/// set, a warning is logged and auth stays disabled to avoid a half-configured gate. +fn resolve_basic_auth( + user: Option<&str>, + password: Option<&str>, +) -> Option<(String, String)> { + match (user, password) { + (Some(u), Some(p)) => { + tracing::info!("HTTP Basic auth enabled for UI/API (user={})", u); + Some((u.to_string(), p.to_string())) + } + (Some(_), None) | (None, Some(_)) => { + tracing::warn!( + "HTTP Basic auth not enabled: both SUBS_BASIC_AUTH_USER and \ + SUBS_BASIC_AUTH_PASSWORD must be set" + ); + None + } + (None, None) => None, + } +} + fn build_rpc_client( rpc_url: &str, rpc_user: Option<&str>, diff --git a/subs/src/routes/auth.rs b/subs/src/routes/auth.rs new file mode 100644 index 0000000..85739f4 --- /dev/null +++ b/subs/src/routes/auth.rs @@ -0,0 +1,117 @@ +//! HTTP Basic authentication middleware for the subsd UI and API. +//! +//! Authentication is only enforced when `AppState::basic_auth` is set (i.e. both +//! `SUBS_BASIC_AUTH_USER` and `SUBS_BASIC_AUTH_PASSWORD` are provided). Health-check +//! endpoints and CORS preflight requests are always allowed through anonymously. + +use axum::{ + body::Body, + extract::{Request, State}, + http::{header, Method, StatusCode}, + middleware::Next, + response::Response, +}; +use base64::Engine; + +use crate::state::AppState; + +/// Split a request path into non-empty segments (leading/trailing slashes ignored). +fn path_segments(path: &str) -> Vec<&str> { + path.split('/').filter(|s| !s.is_empty()).collect() +} + +/// Whether a request may bypass authentication. +/// +/// This covers the liveness probe plus a set of public endpoints used by the +/// handle reservation/claim flow. Matching is method-aware and understands the +/// parameterized routes (`/certs/:handle`, `/spaces/:space/handles/:handle`). +fn is_anonymous(method: &Method, path: &str) -> bool { + let get = *method == Method::GET; + let post = *method == Method::POST; + + match path_segments(path).as_slice() { + // Liveness probe. + ["health"] => get, + // Read-only status used by the public UI. + ["status"] => get, + // Public handle submission / reservation / claim flow. + ["requests"] => post, + ["reserve"] => post, + ["claim"] => post, + // Per-handle certificate lookup: GET /certs/{handle} + ["certs", _handle] => get, + // Per-handle status lookup: GET /spaces/{space}/handles/{subname} + ["spaces", _space, "handles", _subname] => get, + _ => false, + } +} + +/// Constant-time byte comparison to avoid leaking credential length/content via timing. +fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { + return false; + } + let mut diff = 0u8; + for (x, y) in a.iter().zip(b.iter()) { + diff |= x ^ y; + } + diff == 0 +} + +fn credentials_match(req: &Request, user: &str, pass: &str) -> bool { + let Some(value) = req + .headers() + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + else { + return false; + }; + + let Some(encoded) = value + .strip_prefix("Basic ") + .or_else(|| value.strip_prefix("basic ")) + else { + return false; + }; + + let Ok(decoded) = base64::engine::general_purpose::STANDARD.decode(encoded.trim()) else { + return false; + }; + + let expected = format!("{user}:{pass}"); + constant_time_eq(&decoded, expected.as_bytes()) +} + +fn unauthorized() -> Response { + Response::builder() + .status(StatusCode::UNAUTHORIZED) + .header( + header::WWW_AUTHENTICATE, + r#"Basic realm="subs", charset="UTF-8""#, + ) + .body(Body::from("Unauthorized")) + .expect("static unauthorized response is valid") +} + +/// Axum middleware enforcing HTTP Basic auth across the UI and API. +pub async fn require_basic_auth( + State(state): State, + req: Request, + next: Next, +) -> Response { + // Auth disabled unless credentials are configured. + let Some((user, pass)) = state.basic_auth.as_ref() else { + return next.run(req).await; + }; + + // Always allow CORS preflight and the public/anonymous endpoints through. + if req.method() == Method::OPTIONS || is_anonymous(req.method(), req.uri().path()) { + return next.run(req).await; + } + + if credentials_match(&req, user, pass) { + next.run(req).await + } else { + unauthorized() + } +} diff --git a/subs/src/routes/commits.rs b/subs/src/routes/commits.rs index b57ae8e..fc6e14b 100644 --- a/subs/src/routes/commits.rs +++ b/subs/src/routes/commits.rs @@ -260,9 +260,27 @@ pub async fn publish_certs( let (count, remaining) = state .operator - .publish_certs(&space, PUBLISH_BATCH_SIZE, &handles) + .publish_certs( + &space, + PUBLISH_BATCH_SIZE, + &handles, + state.publish_require_finalized, + ) .await - .map_err(|e| json_error(StatusCode::INTERNAL_SERVER_ERROR, e))?; + .map_err(|e| { + let msg = e.to_string(); + if state.publish_require_finalized + && (msg.contains("confirmations") + || msg.contains("not broadcast") + || msg.contains("not confirmed") + || msg.contains("not committed") + || msg.contains("RPC required")) + { + json_error(StatusCode::CONFLICT, msg) + } else { + json_error(StatusCode::INTERNAL_SERVER_ERROR, msg) + } + })?; Ok(Json(PublishResponse { handles_published: count, @@ -372,6 +390,12 @@ pub struct PipelineResponse { pub prover_configured: bool, /// Whether a proving job is currently in flight on the prover pub proving_job_active: bool, + /// When true, publish is blocked until commitments reach 150 confirmations. + pub publish_require_finalized: bool, + /// Whether publishing is currently allowed for unpublished handles. + pub publish_allowed: bool, + /// Reason publish is blocked, when `publish_require_finalized` is enabled. + pub publish_blocked_reason: Option, /// Which proof of the commitment is next (1-based), when proving. pub proof_index: Option, /// How many proofs this commitment needs in total: the first commitment @@ -440,6 +464,20 @@ pub async fn get_pipeline_status( false }; + let publish_require_finalized = state.publish_require_finalized; + let (publish_allowed, publish_blocked_reason) = if !publish_require_finalized || status.unpublished == 0 { + (true, None) + } else { + match state + .operator + .publish_gate(&space_label, publish_require_finalized, PUBLISH_BATCH_SIZE) + .await + { + Ok((allowed, reason)) => (allowed, reason), + Err(e) => (false, Some(e.to_string())), + } + }; + // Ask the prover how far along it is. Only the prover knows, and the value // is stale the moment it is cached, so it is fetched per request rather // than stored. Failures are swallowed: a missing progress bar is a much @@ -453,6 +491,9 @@ pub async fn get_pipeline_status( status, prover_configured, proving_job_active, + publish_require_finalized, + publish_allowed, + publish_blocked_reason, proof_index, proof_total, proving_job_id: active_job_id, diff --git a/subs/src/routes/console.rs b/subs/src/routes/console.rs index 9e1376a..bdbd4aa 100644 --- a/subs/src/routes/console.rs +++ b/subs/src/routes/console.rs @@ -8,9 +8,35 @@ use axum::{ }; use serde::{Deserialize, Serialize}; +use reqwest::RequestBuilder; + use crate::state::AppState; use super::json_error; +/// Apply Spaced RPC credentials to an outbound request (same precedence as `build_rpc_client`). +fn apply_spaced_rpc_auth( + req: RequestBuilder, + state: &AppState, +) -> Result { + if let Some(user) = state.spaced_rpc_user.as_deref() { + return Ok(req.basic_auth(user, state.spaced_rpc_password.as_deref())); + } + if let Some(path) = state.spaced_rpc_cookie.as_ref() { + let cookie = std::fs::read_to_string(path).map_err(|e| { + json_error( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to read RPC cookie file: {e}"), + ) + })?; + let encoded = base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + cookie.trim().as_bytes(), + ); + return Ok(req.header("Authorization", format!("Basic {encoded}"))); + } + Ok(req) +} + #[derive(Debug, Deserialize)] pub struct RpcRequest { pub method: String, @@ -84,7 +110,7 @@ pub async fn proxy_spaced( .as_ref() .ok_or_else(|| json_error(StatusCode::SERVICE_UNAVAILABLE, "Spaced RPC URL not configured"))?; - proxy_rpc_call(rpc_url, &request, Some(("user", "pass"))).await + proxy_rpc_call(rpc_url, &request, |req| apply_spaced_rpc_auth(req, &state)).await } /// POST /rpc/bitcoin - Proxy RPC call to bitcoind (test-rig only) @@ -97,7 +123,10 @@ pub async fn proxy_bitcoin( .as_ref() .ok_or_else(|| json_error(StatusCode::SERVICE_UNAVAILABLE, "Bitcoin RPC not available (only in test-rig mode)"))?; - proxy_rpc_call(rpc_url, &request, Some(("user", "password"))).await + proxy_rpc_call(rpc_url, &request, |req| { + Ok(req.basic_auth("user", Some("password"))) + }) + .await } /// POST /rpc/mine - Mine blocks (test-rig only) @@ -130,11 +159,14 @@ pub async fn mine_blocks( Err(json_error(StatusCode::SERVICE_UNAVAILABLE, "Mining only available in test-rig mode")) } -async fn proxy_rpc_call( +async fn proxy_rpc_call( rpc_url: &str, request: &RpcRequest, - auth: Option<(&str, &str)>, -) -> Result, Response> { + apply_auth: F, +) -> Result, Response> +where + F: FnOnce(RequestBuilder) -> Result, +{ let client = reqwest::Client::new(); // Build JSON-RPC request @@ -145,14 +177,11 @@ async fn proxy_rpc_call( "params": request.params, }); - let mut req = client + let req = client .post(rpc_url) .header("Content-Type", "application/json") .json(&rpc_body); - - if let Some((user, pass)) = auth { - req = req.basic_auth(user, Some(pass)); - } + let req = apply_auth(req)?; let response = req .timeout(std::time::Duration::from_secs(30)) diff --git a/subs/src/routes/mod.rs b/subs/src/routes/mod.rs index e9e3cfb..6284b94 100644 --- a/subs/src/routes/mod.rs +++ b/subs/src/routes/mod.rs @@ -1,5 +1,6 @@ //! Route handlers for the subsd REST API. +pub mod auth; pub mod certs; pub mod commits; pub mod config; @@ -34,6 +35,8 @@ pub fn router() -> Router { .route("/ui/transactions", get(web::transactions_page)) .route("/ui/spaces/:space", get(web::space_page)) .route("/ui/spaces/:space/handles/:handle", get(web::handle_page)) + // Health probe (kept anonymous by the auth middleware) + .route("/health", get(status::health)) // API: Status & Spaces .route("/status", get(status::get_status)) .route("/spaces", get(status::list_spaces)) diff --git a/subs/src/routes/status.rs b/subs/src/routes/status.rs index 5b46e3e..d4ede4a 100644 --- a/subs/src/routes/status.rs +++ b/subs/src/routes/status.rs @@ -12,6 +12,11 @@ use subs_core::{HandlesListResult, SpaceStatus, StatusResult}; use crate::state::AppState; use super::json_error; +/// GET /health - Lightweight liveness probe (always anonymous). +pub async fn health() -> &'static str { + "ok" +} + /// GET /status - Get status of all spaces pub async fn get_status( State(state): State, diff --git a/subs/src/state.rs b/subs/src/state.rs index 2868d71..16f3d23 100644 --- a/subs/src/state.rs +++ b/subs/src/state.rs @@ -1,5 +1,6 @@ //! Application state for the subsd server. +use std::path::PathBuf; use std::sync::Arc; use subs_core::Operator; @@ -16,6 +17,16 @@ pub struct AppState { pub config: Arc, /// Spaced RPC URL for the console pub spaced_rpc_url: Option, + /// Spaced RPC username for proxied calls + pub spaced_rpc_user: Option, + /// Spaced RPC password for proxied calls + pub spaced_rpc_password: Option, + /// Spaced RPC cookie file for proxied calls (used when user/password not set) + pub spaced_rpc_cookie: Option, + /// HTTP Basic auth credentials (user, password); auth is enforced when Some + pub basic_auth: Option<(String, String)>, + /// When true, publish is blocked until commitments reach 150 on-chain confirmations. + pub publish_require_finalized: bool, /// Bitcoin RPC URL (only available in test-rig mode) pub bitcoin_rpc_url: Option, /// Certrelay URL (only available in test-rig mode) @@ -27,32 +38,54 @@ pub struct AppState { impl AppState { #[cfg(not(feature = "test-rig"))] + #[allow(clippy::too_many_arguments)] pub fn with_rpc_urls( operator: Operator, config: ConfigStore, spaced_rpc_url: Option, + spaced_rpc_user: Option, + spaced_rpc_password: Option, + spaced_rpc_cookie: Option, + basic_auth: Option<(String, String)>, + publish_require_finalized: bool, _bitcoin_rpc_url: Option, ) -> Self { Self { operator: Arc::new(operator), config: Arc::new(config), spaced_rpc_url, + spaced_rpc_user, + spaced_rpc_password, + spaced_rpc_cookie, + basic_auth, + publish_require_finalized, bitcoin_rpc_url: None, certrelay_url: None, } } #[cfg(feature = "test-rig")] + #[allow(clippy::too_many_arguments)] pub fn with_rpc_urls( operator: Operator, config: ConfigStore, spaced_rpc_url: Option, + spaced_rpc_user: Option, + spaced_rpc_password: Option, + spaced_rpc_cookie: Option, + basic_auth: Option<(String, String)>, + publish_require_finalized: bool, bitcoin_rpc_url: Option, ) -> Self { Self { operator: Arc::new(operator), config: Arc::new(config), spaced_rpc_url, + spaced_rpc_user, + spaced_rpc_password, + spaced_rpc_cookie, + basic_auth, + publish_require_finalized, bitcoin_rpc_url, certrelay_url: None, test_rig: None, @@ -60,10 +93,16 @@ impl AppState { } #[cfg(feature = "test-rig")] + #[allow(clippy::too_many_arguments)] pub fn with_test_rig( operator: Operator, config: ConfigStore, spaced_rpc_url: Option, + spaced_rpc_user: Option, + spaced_rpc_password: Option, + spaced_rpc_cookie: Option, + basic_auth: Option<(String, String)>, + publish_require_finalized: bool, bitcoin_rpc_url: Option, certrelay_url: Option, test_rig: Arc, @@ -72,6 +111,11 @@ impl AppState { operator: Arc::new(operator), config: Arc::new(config), spaced_rpc_url, + spaced_rpc_user, + spaced_rpc_password, + spaced_rpc_cookie, + basic_auth, + publish_require_finalized, bitcoin_rpc_url, certrelay_url, test_rig: Some(test_rig), diff --git a/subs/templates/base.html b/subs/templates/base.html index 2db8662..d9c3a86 100644 --- a/subs/templates/base.html +++ b/subs/templates/base.html @@ -197,6 +197,13 @@ transition: background 0.15s; } .publish-btn:hover { background: #eb7e3a; } +.publish-btn:disabled { + background: var(--border-subtle); + color: var(--text-muted); + cursor: not-allowed; + opacity: 0.75; +} +.publish-btn:disabled:hover { background: var(--border-subtle); } .publish-btn .count { background: rgba(0,0,0,0.18); padding: 1px 6px; border-radius: 3px; font-size: 10px; diff --git a/subs/templates/space.html b/subs/templates/space.html index be249f0..6d0766d 100644 --- a/subs/templates/space.html +++ b/subs/templates/space.html @@ -308,7 +308,7 @@

Handles

const msg = $('pipelineMessage'); const actions = $('pipelineActions'); - updatePublishBar(j.unpublished || 0); + updatePublishBar(j); // Re-arm before the has_pending branch below: publishing no longer // sets has_pending, so that branch returns early and would otherwise @@ -565,16 +565,46 @@

Handles

el.innerHTML = h; } -function updatePublishBar(n) { +function canPublish(pipeline) { + if (!pipeline || !(pipeline.unpublished > 0)) return false; + if (!pipeline.publish_require_finalized) return true; + return pipeline.publish_allowed !== false; +} + +/** Human-readable reason publish is blocked, including remaining confirmations when known. */ +function publishBlockedReason(pipeline) { + if (!pipeline) { + return 'Publishing is blocked until the commitment has 150 confirmations'; + } + if (pipeline.publish_blocked_reason) return pipeline.publish_blocked_reason; + const m = String(pipeline.message || '').match(/^(\d+)\/(\d+)\s+confirmations$/); + if (m) { + const cur = parseInt(m[1], 10); + const max = parseInt(m[2], 10); + const remaining = Math.max(0, max - cur); + return `Publishing is blocked until finalization: ${cur}/${max} confirmations (${remaining} more needed)`; + } + return 'Publishing is blocked until the commitment has 150 confirmations'; +} + +function updatePublishBar(pipeline) { const bar = $('publishBar'); - if (n > 0) { - const batch = Math.min(n, 100); - const label = batch < n ? `${batch}/${n}` : `${n}`; - bar.innerHTML = `⊕ Publish ${label}`; - bar.classList.remove('hidden'); - } else { + const n = pipeline?.unpublished || 0; + if (n <= 0) { bar.classList.add('hidden'); + bar.disabled = false; + bar.title = ''; + return; } + + const batch = Math.min(n, 100); + const label = batch < n ? `${batch}/${n}` : `${n}`; + bar.innerHTML = `⊕ Publish ${label}`; + bar.classList.remove('hidden'); + + const allowed = canPublish(pipeline); + bar.disabled = !allowed; + bar.title = allowed ? '' : publishBlockedReason(pipeline); } // --------------------------------------------------------------------------- @@ -648,6 +678,10 @@

Handles

} async function publishCerts() { + if (currentPipeline && !canPublish(currentPipeline)) { + logAction(`Error: ${publishBlockedReason(currentPipeline)}`); + return; + } logAction('Publishing certificates...'); try { const { ok, data } = await api(`${spaceUrl}/publish`, { method: 'POST' }); @@ -980,7 +1014,10 @@

Handles

else h += btn('Park', 'park') + btn('Unpark', 'unpark'); h += btn('Remove', 'remove'); } - h += btn(allPublished && !nonePublished ? 'Republish' : 'Publish', 'publish'); + const publishAllowed = !currentPipeline?.publish_require_finalized || currentPipeline?.publish_allowed !== false; + if (publishAllowed) { + h += btn(allPublished && !nonePublished ? 'Republish' : 'Publish', 'publish'); + } btns.innerHTML = h; } @@ -999,6 +1036,10 @@

Handles

} async function bulkAction(action) { + if (action === 'publish' && currentPipeline && !canPublish(currentPipeline)) { + logAction(`Error: ${publishBlockedReason(currentPipeline)}`); + return; + } if (action === 'remove') { const count = selectAllMode ? lastTotal : selectedHandles.size; if (!confirm(`Remove ${count} staged handle(s)? This cannot be undone.`)) return; diff --git a/tools/wif-to-hex/Cargo.toml b/tools/wif-to-hex/Cargo.toml new file mode 100644 index 0000000..d8edd67 --- /dev/null +++ b/tools/wif-to-hex/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "wif-to-hex" +version = "0.1.0" +edition = "2021" +description = "Decode Bitcoin WIF to a 64-character hex private key" + +[[bin]] +name = "wif-to-hex" +path = "src/main.rs" + +[dependencies] +anyhow = { workspace = true } +bitcoin = { workspace = true } +clap = { workspace = true } +hex = { workspace = true } diff --git a/tools/wif-to-hex/src/main.rs b/tools/wif-to-hex/src/main.rs new file mode 100644 index 0000000..2f6111b --- /dev/null +++ b/tools/wif-to-hex/src/main.rs @@ -0,0 +1,31 @@ +//! Decode a Bitcoin WIF private key to 64 hex characters (32-byte secret). + +use anyhow::{Context, Result}; +use bitcoin::PrivateKey; +use clap::Parser; + +#[derive(Parser)] +#[command( + name = "wif-to-hex", + about = "Decode Bitcoin WIF to a 64-character hex private key" +)] +struct Cli { + /// WIF private key (mainnet K/L/5 prefix, or testnet c/9 prefix) + wif: String, + + /// Prefix output with 0x + #[arg(long)] + prefix: bool, +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let pk = PrivateKey::from_wif(cli.wif.trim()) + .context("invalid WIF (bad Base58Check encoding or checksum)")?; + let hex = hex::encode(pk.inner.secret_bytes()); + if cli.prefix { + print!("0x"); + } + println!("{hex}"); + Ok(()) +}