From 00a2ffba4915d2ff412475ce1970bc884fa0db4e Mon Sep 17 00:00:00 2001 From: Nick the Sick Date: Tue, 11 Aug 2026 13:25:53 +0200 Subject: [PATCH 1/9] feat: personal library over XRPC with atproto service auth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes the whole personal library reachable over /xrpc/*, authenticated with atproto inter-service auth so a script or e-reader can use it rather than only a browser session. Asked for in https://bsky.md/profile/pds.dad/post/3msrmxae2tk2g. Binary bodies: uploadPersonalBook declares an explicit ebook MIME allowlist instead of */* (which made @atcute/lex-cli emit no validator at all) and takes `filename` as a required param, replacing an x-file-name header that defaulted to "unknown". application/octet-stream is in the list deliberately — the iOS app, mobile pickers and curl all send it; detectFormat checking magic bytes against the extension is the real gate. OPDS parity: new getPersonalBookFile and getPersonalBookCover (blob outputs owning their own ETag/304), new listPersonalShelves (the root call — /opds root was 431 of 551 OPDS hits in 48h of production traffic), and q/sort/storage on getPersonalLibrary. Service auth via ServiceJwtVerifier. Methods declare auth: "identity" | "pdsWrite" and the registration wrapper derives the lxm from the schema's NSID, so a method's route and its token binding cannot drift. pdsWrite refuses service auth outright — it proves key control, not that we hold an OAuth grant. isKnownAccount gates it so an arbitrary DID can't open a quota's worth of disk. One upload implementation. processBookUpload and /library/upload had drifted in four ways; both are now thin adapters over src/utils/uploadPersonalBook.ts. That closed real memory problems: bodyLimit() buffered chunked bodies twice, and the epub/cbz parsers inflated every image in an archive to keep one cover. Worst-case upload RSS goes from unbounded to a bounded ~630 MB cluster-wide. Storage quota (2 GB), enforced as a SUM evaluated inside the INSERT so two concurrent uploads can't both slip past. Verified safe against production: the heaviest user holds 39 MB. Also fixes the KOSync progress bridge (an upload before the first sync left sync_document.hiveId null forever), persists admin backfill progress across restarts, and gives OPDS covers an ETag — production served 43 cover fetches in 48h and could never answer a single one with a 304. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 350 +++++++++- bun.lock | 1 + bunfig.toml | 6 + lexicons/auth.json | 26 +- lexicons/getPersonalBookCover.json | 36 + lexicons/getPersonalBookFile.json | 29 + lexicons/getPersonalLibrary.json | 48 +- lexicons/listPersonalShelves.json | 36 + lexicons/uploadPersonalBook.json | 50 +- package.json | 1 + src/app.ts | 38 +- src/auth/client.ts | 34 +- src/auth/router.tsx | 7 + src/auth/session.test.ts | 14 +- src/auth/token-refresh.test.ts | 9 +- src/bsky/id-resolver.ts | 30 +- src/bsky/lexicon/generated/index.ts | 3 + .../buzz/bookhive/getPersonalBookCover.ts | 40 ++ .../buzz/bookhive/getPersonalBookFile.ts | 34 + .../types/buzz/bookhive/getPersonalLibrary.ts | 60 +- .../buzz/bookhive/listPersonalShelves.ts | 41 ++ .../types/buzz/bookhive/uploadPersonalBook.ts | 38 +- src/client/components/LibraryManager.tsx | 68 +- src/context.ts | 83 +++ src/db.ts | 93 +++ src/env.ts | 30 +- src/pages/library.test.tsx | 39 ++ src/pages/library.tsx | 40 +- src/routes/admin.ts | 7 +- src/routes/library.test.ts | 76 +- src/routes/library.tsx | 278 +++----- src/routes/opds.test.ts | 37 + src/routes/opds.ts | 33 +- src/routes/sync/kosync.test.ts | 283 ++++++++ src/routes/sync/kosync.ts | 25 +- src/test/env-setup.ts | 22 + src/types.ts | 6 + src/utils/account.ts | 69 ++ src/utils/bookMatching.ts | 94 +++ src/utils/bookMetadata/cbz.ts | 36 +- src/utils/bookMetadata/cover.ts | 8 + src/utils/bookMetadata/epub.ts | 71 +- src/utils/bookMetadata/hash.ts | 26 + src/utils/bookMetadata/index.ts | 21 +- src/utils/bookMetadata/testFixtures.ts | 148 ++++ src/utils/catalogBookService.test.ts | 83 +++ src/utils/catalogBookService.ts | 52 +- src/utils/filenameMatching.test.ts | 176 +++++ src/utils/filenameMatching.ts | 315 +++++++++ src/utils/personalLibrary.ts | 51 +- src/utils/syncMatching.test.ts | 351 ++++++++++ src/utils/syncMatching.ts | 344 ++++++++- src/utils/uploadPersonalBook.test.ts | 592 ++++++++++++++++ src/utils/uploadPersonalBook.ts | 535 ++++++++++++++ src/xrpc/auth.test.ts | 330 +++++++++ src/xrpc/auth.ts | 92 +++ src/xrpc/personalLibrary.test.ts | 505 ++++++++++++++ src/xrpc/replay-store.ts | 58 ++ src/xrpc/router.ts | 653 +++++++++++------- 59 files changed, 6092 insertions(+), 569 deletions(-) create mode 100644 lexicons/getPersonalBookCover.json create mode 100644 lexicons/getPersonalBookFile.json create mode 100644 lexicons/listPersonalShelves.json create mode 100644 src/bsky/lexicon/generated/types/buzz/bookhive/getPersonalBookCover.ts create mode 100644 src/bsky/lexicon/generated/types/buzz/bookhive/getPersonalBookFile.ts create mode 100644 src/bsky/lexicon/generated/types/buzz/bookhive/listPersonalShelves.ts create mode 100644 src/routes/sync/kosync.test.ts create mode 100644 src/test/env-setup.ts create mode 100644 src/utils/account.ts create mode 100644 src/utils/bookMatching.ts create mode 100644 src/utils/bookMetadata/testFixtures.ts create mode 100644 src/utils/catalogBookService.test.ts create mode 100644 src/utils/filenameMatching.test.ts create mode 100644 src/utils/filenameMatching.ts create mode 100644 src/utils/syncMatching.test.ts create mode 100644 src/utils/uploadPersonalBook.test.ts create mode 100644 src/utils/uploadPersonalBook.ts create mode 100644 src/xrpc/auth.test.ts create mode 100644 src/xrpc/auth.ts create mode 100644 src/xrpc/personalLibrary.test.ts create mode 100644 src/xrpc/replay-store.ts diff --git a/AGENTS.md b/AGENTS.md index 3bdeb5a1..717d3e7a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -147,7 +147,17 @@ Two traps this encodes, both of which caused real bugs: - `/healthcheck` → JSON status + git sha - `/metrics` → Prometheus -- `/admin/*` → `src/routes/admin.ts` (gated by `EXPORT_SHARED_SECRET`) +- `/admin/*` → `src/routes/admin.ts` (gated by `EXPORT_SHARED_SECRET`). + `GET /admin/backfill-catalog/progress` reads **through the KV**, not just this + process's memory: the backfill runs for hours on the primary worker while the + request lands on any of the three, so an in-memory-only answer reported `idle` + for a live job and lost the outcome of a finished one. A _stored_ `running` + can only mean the process died mid-run (a live run answers from memory), so it + is reported as `interrupted`. + **Persist the object, never `JSON.stringify` of it.** unstorage runs `destr` + over whatever a driver returns, so a stored JSON string reads back as an + object and any `JSON.parse` of it throws — which silently discarded every + stored run. Same idiom as `enqueuePdsWrite`. - `/debug/*` → `src/routes/debug.ts` (gated by `EXPORT_SHARED_SECRET`) - `/import` (POST `/goodreads`, `/storygraph`) → `src/routes/import.ts` — CSV import handler @@ -221,7 +231,7 @@ User book lists ("shelves"). Uses **popfeed** lexicons (`social.popfeed.feed.lis Personal library: ebook uploads, e-reader credentials, sync documents. All auth-required. - GET `/` → `src/pages/library.tsx` -- POST `/upload` → multipart upload (validates format, computes KOReader partial MD5, parses metadata, auto-links matching `sync_document`). Content-negotiated: JSON for mobile, 302 for browser. Size checked before `arrayBuffer()`. +- POST `/upload` → multipart upload. A **thin adapter** over `uploadPersonalBook` (`src/utils/uploadPersonalBook.ts`) — the same core the XRPC procedure calls. Content-negotiated: JSON for mobile, `302 /library?error=` for browsers (a plain `
` can't read a JSON error body, so the reason round-trips as a code `LibraryPage` renders as an alert). **No `bodyLimit()` middleware** — see the upload-core section for why it was the worst memory path in the codebase. - GET `/covers/:hash` → cover image; GET `/books/:hash/download` → file download (shares `streamPersonalBook` with OPDS) - GET `/shelves` → JSON shelf list with counts - GET `/sync/password`, POST `/sync/rotate` → KOSync password (duplicated from settings) @@ -260,6 +270,66 @@ Auth: `x-auth-user` (handle) + `x-auth-key` (md5 of HMAC-derived password). Prog - PUT `/syncs/progress` → push progress; GET `/syncs/progress/:document` → pull progress - GET `/syncs/documents` → list all synced documents +**A KOSync `document` id is not necessarily a content hash.** KOReader's +checksum method is a user setting: `BINARY` (the default) is +`koreaderPartialMD5` over the file, but `FILENAME` is plain `md5(basename)`, and +users switch to it precisely because their files are _not_ byte-identical across +devices. Matching only `documentHash = personal_book.contentHash` therefore +never fired for any of them. `SAME_BOOK_FILE` (`src/utils/syncMatching.ts`) is +the one predicate for "same book" — content hash, filename hash, or normalized +filename — and it is used as a **correlated subquery, never a join**, because a +document can match several files (and vice versa) and a join fans that out into +duplicate rows that also break `getPersonalLibrary`'s pagination. + +Separately, the payload may carry `metadata: { filename, title, authors }` +(KOReader PR #15306, merged 2026-04-29 — its "Send document metadata" toggle +**defaults off**, so most KOReader users still send none of it; CrossPoint sends +it). Two non-obvious things about that object, each of which silently matches +nothing if you assume otherwise: + +- **`authors` is newline-separated**, not comma- or tab-separated. It is + `doc_props.authors`, one of the three props KOReader's metadata editor opens + with `allow_newline = true`. (Three separators are in play across the app: + newline from KOReader, **comma** in `personal_book.authors` from `parseBook`, + tab in `hive_book.authors`.) +- **`title` may itself be a filename.** It is `doc_props.display_title`, defined + as `props.title or splitFileNameType(filepath)` — so any document with no + embedded title sends the filename stem, dashes and all. `matchSyncDocument` + runs the client's `title` through the filename parser for that reason. + +**The routes call `matchSyncDocumentForUser`, not `matchSyncDocument`.** With +both KOReader defaults in force — BINARY checksum, `send_metadata` off, which is +most users — the entire request identifies the book as one partial-MD5 hash and +nothing else, so matching the _payload_ is hopeless no matter how good the tiers +get. But that hash is `personal_book.contentHash`: if the user uploaded the +file, we already parsed real title/author metadata out of the ebook at upload +time and may already have resolved it to a book. +`matchSyncDocumentForUser` finds the file first, inherits its `hiveId`, else +matches on the file's own metadata, and writes the result back onto the file +(plus `user_book.owned`). `uploadPersonalBook` already pushes a link the other +way when the document exists first; this covers the opposite ordering. + +`matchSyncDocument` itself runs three tiers, and the invariant across all of +them is that **a wrong link is worse than no link**: it writes progress onto a +book the user isn't reading and mirrors it to their PDS, while a miss just +leaves the document for them to link by hand. + +1. Exact `hive_book.id` hash of the client's title+author. +2. Exact id hash of title/author pairs parsed out of the filename. Both + orderings of an `A - B` split are tried; that is safe _because_ it resolves + against the catalogue, so a wrong guess hashes to an id that does not exist. +3. Fuzzy. Candidates come from `hive_book_fts`, searched by **author** as well + as by title — FTS matches phrases, so a title search alone can never reach + "The Hitchhiker's Guide" from "Hitchhikers Guide", whereas an author's name + is spelled the same either way and their books can then be compared in JS. + Acceptance is `titlesEquivalent` (equal content-word sets, gated **both** + ways — one-way containment would accept "Dune" as "Dune Messiah") plus an + agreeing author. With no author signal anywhere, only a title that names + exactly one book is accepted. + +Do not gate any of this on title/author being present — that gate is what made a +filename-only client unmatchable in the first place. + ### Shared route helpers `src/routes/lib.ts` — `cacheControl`, `searchBooks`, `ensureBookIdentifiersCurrent`, `refetchBooks`, `refetchBuzzes`, `refetchLists`, `syncFollowsIfNeeded`. @@ -382,7 +452,7 @@ Client hooks/utils: `useSearchBooks.ts`, `useDebounce.ts`, `icons.tsx`, `debounc ### Database (`src/db.ts`) -SQLite via Kysely. Schema + all migrations (001–021) in one file. `createDb` sets WAL/perf PRAGMAs. `mmap_size` defaults to 0 (see `DB_MMAP_SIZE` in `src/env.ts`). Kysely talks to `bun:sqlite` through `src/bun-sqlite-kysely.ts`, which rewrites `begin` to `BEGIN IMMEDIATE` (deferred transactions fail with `SQLITE_BUSY_SNAPSHOT` across cluster processes). +SQLite via Kysely. Schema + all migrations (001–023) in one file. `createDb` sets WAL/perf PRAGMAs. `mmap_size` defaults to 0 (see `DB_MMAP_SIZE` in `src/env.ts`). Kysely talks to `bun:sqlite` through `src/bun-sqlite-kysely.ts`, which rewrites `begin` to `BEGIN IMMEDIATE` (deferred transactions fail with `SQLITE_BUSY_SNAPSHOT` across cluster processes). | Table | Purpose | Key columns | | --------------------- | ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -396,9 +466,9 @@ SQLite via Kysely. Schema + all migrations (001–021) in one file. `createDb` s | `user_follows` | Cached follow graph | userDid, followsDid, followedAt, syncedAt, **isActive** | | `book_list` | User-created book lists | **uri (PK, AT URI)**, userDid, name, description, ordered, tags, createdAt | | `book_list_item` | Items in a book list | **uri (PK, AT URI)**, userDid, **listUri**, hiveId, position | -| `sync_document` | E-reader sync progress | id (PK), userDid, provider, documentHash (UNIQUE per user+provider), hiveId (nullable), filename, title, authors, progressData (JSON) | +| `sync_document` | E-reader sync progress | id (PK), userDid, provider, documentHash (UNIQUE per user+provider), hiveId (nullable), filename, **filenameKey**, title, authors, progressData (JSON) | | `enrich_queue` | Pending Goodreads enrich | **hiveId (PK — the dedupe)**, **enqueuedAt** (age ceiling; survives re-enqueue), attempts, nextAttemptAt, claimedAt, lastError | -| `personal_book` | Uploaded ebook files | contentHash (PK), userDid, filename, title, authors, format, hiveId (nullable), fileSize | +| `personal_book` | Uploaded ebook files | contentHash (PK), userDid, filename, **filenameHash**, **filenameKey**, title, authors, format, hiveId (nullable), fileSize | | `personal_shelf` | User's personal shelves | id (PK, autoincrement), userDid, name, description | | `personal_shelf_item` | Books in personal shelves | shelfId, contentHash (PK pair) | @@ -432,37 +502,116 @@ the primary worker inside the startup barrier where VACUUM already runs. ### KV Cache (`src/sqlite-kv.ts`) -SQLite-backed unstorage. Mounts: `search:` (in-memory LRU), `profile:`, `identity:`, `follows_sync:`, `auth_session:`, `auth_state:`, `book_lock:`, `sync_pending:`, `sync_token:`, `page:` (anon page cache). VACUUMed on startup by primary worker; incremental vacuum on 15-min sweep. +SQLite-backed unstorage. Mounts: `search:` (in-memory LRU), `profile:`, `identity:`, `follows_sync:`, `auth_session:`, `auth_state:`, `book_lock:`, `sync_pending:`, `sync_token:`, `account:` (service-auth gate — see Auth), `page:` (anon page cache). VACUUMed on startup by primary worker; incremental vacuum on 15-min sweep. The `svc_jti` table (service-auth replay store) lives in the same file but is written directly rather than through unstorage — `src/xrpc/replay-store.ts`. ### Key Utilities (`src/utils/`) -| File | Purpose | -| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `getBook.ts` | Book record CRUD against user's PDS | -| `getProfile.ts` | Profile fetching from Bluesky | -| `getFollows.ts` | Follow graph sync | -| `enrichBookData.ts` | Goodreads enrichment (semaphore-bounded, 45s deadline) | -| `enrichQueue.ts` | `enrich_queue` producer + primary-worker drain, and the `retry`/`defer`/`dead` accounting. The `exhausted` gauge/heartbeat counts `hive_book.enrichFailedAt` inside the cooldown, **not** queue rows at MAX_ATTEMPTS — those are deleted as they exhaust, so that read was always 0. `deferred` counts books parked on something that isn't their fault; it replaced `circuit_open` as the signal that fetching is in trouble | -| `semaphore.ts` | Async concurrency limiter + `withTimeout` | -| `circuitBreaker.ts` | Three-state breaker — **`auth/restore-guard.ts` only.** Right when refusing is cheaper for a waiting user than failing; wrong for scraping, where the queue can defer instead. See the note under Scrapers | -| `bookIdentifiers.ts` | ISBN/ID normalization + persistence | -| `bookProgress.ts` | BookProgress serialization | -| `readThroughCache.ts` | KV read-through with TTL | -| `csv.ts` | Goodreads/StoryGraph CSV parsers | -| `lists.ts` | Book list (shelf) CRUD against PDS | -| `readingStats.ts` | Reading stats aggregation by year | -| `imageProxy.ts` | imgproxy signing + proxy helper | -| `personalLibrary.ts` | Personal library paths, `streamPersonalBook` | -| `bookMetadata/` | Ebook metadata parsing (epub, mobi, fb2, cbz, cover extraction, KOReader hash) | -| `bookMeta.ts` | Book metadata utilities | -| `syncMatching.ts` | KOReader document → BookHive book matching; `NO_HIVE_MATCH` sentinel | -| `syncBridge.ts` | Bridge e-reader progress → user_book + queue PDS write | -| `ftsQuery.ts` | FTS5 MATCH expression builder | -| `importBook.ts` | Import a single book record | -| `authorMatching.ts` | Author name matching | -| `manifest.ts` | Vite manifest → asset URLs | -| `xml.ts` | XML utilities | -| Other | `getLanguages.ts`, `catalogBookService.ts`, `deleteAccount.ts`, `dbExport.ts`, `generateInitialsAvatar.ts`, `htmlToText.ts`, `batchTransform.ts`, `lazy.ts`, `hiveBookGenres.ts`, `ensureBookCataloged.ts`, `uploadImageBlob.ts` | +| File | Purpose | +| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `getBook.ts` | Book record CRUD against user's PDS | +| `getProfile.ts` | Profile fetching from Bluesky | +| `getFollows.ts` | Follow graph sync | +| `enrichBookData.ts` | Goodreads enrichment (semaphore-bounded, 45s deadline) | +| `enrichQueue.ts` | `enrich_queue` producer + primary-worker drain, and the `retry`/`defer`/`dead` accounting. The `exhausted` gauge/heartbeat counts `hive_book.enrichFailedAt` inside the cooldown, **not** queue rows at MAX_ATTEMPTS — those are deleted as they exhaust, so that read was always 0. `deferred` counts books parked on something that isn't their fault; it replaced `circuit_open` as the signal that fetching is in trouble | +| `semaphore.ts` | Async concurrency limiter + `withTimeout` | +| `circuitBreaker.ts` | Three-state breaker — **`auth/restore-guard.ts` only.** Right when refusing is cheaper for a waiting user than failing; wrong for scraping, where the queue can defer instead. See the note under Scrapers | +| `bookIdentifiers.ts` | ISBN/ID normalization + persistence | +| `bookProgress.ts` | BookProgress serialization | +| `readThroughCache.ts` | KV read-through with TTL | +| `csv.ts` | Goodreads/StoryGraph CSV parsers | +| `lists.ts` | Book list (shelf) CRUD against PDS | +| `readingStats.ts` | Reading stats aggregation by year | +| `imageProxy.ts` | imgproxy signing + proxy helper | +| `personalLibrary.ts` | Personal library paths, `streamPersonalBook`, `getStorageUsage`/`getStorageQuota` | +| `uploadPersonalBook.ts` | **The one** "put this ebook in this user's library". Both `POST /library/upload` and the XRPC procedure are thin adapters over it — see below | +| `account.ts` | `isKnownAccount`/`markAccount` — the gate on service auth | +| `bookMetadata/` | Ebook metadata parsing (epub, mobi, fb2, cbz, cover extraction, KOReader hash) | +| `bookMeta.ts` | Book metadata utilities | +| `syncMatching.ts` | KOReader document → BookHive book matching (3 tiers, see below); `NO_HIVE_MATCH` sentinel; `SAME_BOOK_FILE` | +| `filenameMatching.ts` | Filename-derived identity: `koreaderFilenameHash`, `filenameKey`, `filenameBookCandidates`, `titlesEquivalent`, `authorsMatch` | +| `bookMatching.ts` | Fuzzy title scoring primitives (`similarityScore`, `contentWords`, `contentWordsMatch`), ported from MIT-licensed shelfcheck | +| `syncBridge.ts` | Bridge e-reader progress → user_book + queue PDS write | +| `ftsQuery.ts` | FTS5 MATCH expression builder | +| `importBook.ts` | Import a single book record | +| `authorMatching.ts` | Author name matching | +| `manifest.ts` | Vite manifest → asset URLs | +| `xml.ts` | XML utilities | +| Other | `getLanguages.ts`, `catalogBookService.ts`, `deleteAccount.ts`, `dbExport.ts`, `generateInitialsAvatar.ts`, `htmlToText.ts`, `batchTransform.ts`, `lazy.ts`, `hiveBookGenres.ts`, `ensureBookCataloged.ts`, `uploadImageBlob.ts` | + +### The upload core (`src/utils/uploadPersonalBook.ts`) + +**There is exactly one upload implementation. Do not add a second.** There used +to be two — the live multipart route and a `processBookUpload` in the XRPC +router whose own doc comment claimed to be the shared core — and they drifted in +four ways (cover validation, sync matching, `sync_document.hiveId` writeback, +empty-string handling) before anyone noticed. Both routes are now thin adapters. + +**The step order is the design, not an accident**, and each step is where it is +for a measured reason: + +1. Reject on the _declared_ size and against the quota — before a byte is read. +2. `writeCapped` streams the body to `{libraryDir}/.tmp/*.part` at **~64 KB of + RSS**, capping as it goes. This is what replaced `bodyLimit()`, which only + short-circuits on `Content-Length`: given a chunked body it drained the whole + stream into an array and rebuilt the Request, so a compliant 100 MB chunked + upload was buffered there **and again** by `formData()`. +3. `detectFormat` off a **4 KB head** (it reads ≤512 bytes, plus 60–68 for MOBI). +4. `koreaderPartialMD5File` reads **twelve 1 KB windows** — the hash never needed + the whole file; the buffer was incidental to how the old path worked. +5. **Duplicate check before the parse**, so a re-upload allocates nothing. + `src/routes/library.test.ts` leans on this ordering to exercise the duplicate + path without touching the library directory. +6. `parseBook` inside `parseSemaphore` — the **only** full-buffer step, because + fflate reads a ZIP's central directory from the end of one contiguous array. + `UPLOAD_PARSE_CONCURRENCY` (2) is therefore the memory bound on uploads, and + it is **per process**: at the deployed `WEB_CONCURRENCY=3` the ceiling is + `2 × 3 × 100 MB ≈ 630 MB`, against a previously _unbounded_ ~200–400 MB per + in-flight upload. Production baseline is ~1.05 GB RSS across the three + workers in a 6 GB container, so that headroom fits comfortably. +7. Cover gated on `isUsableCover`, always. `coverPath IS NOT NULL` is the only + signal driving `coverUrl` on the web library, the OPDS feed and the XRPC book + view, so an unvalidated cover is a dead URL and a blank box in all three. +8. **The quota `SUM` is evaluated inside the INSERT's `WHERE`.** SQLite + serialises writers, so this is exact — two concurrent uploads cannot both + observe the pre-insert total. A per-process mutex would not have worked: + production is four independent processes against one file. +9. `rename` into place — same filesystem, zero bytes copied. The row commits + _before_ the bytes move, which beats writing 100 MB and then discovering a + problem. + +Sync-doc linking is **exact first, fuzzy only on a miss**. The XRPC path used to +run `matchSyncDocument` first, which let a title/author guess beat a byte-exact +`documentHash` — wrong, and it paid for up to four FTS queries on the common +path. Newly-linked documents also get their **already-recorded progress +bridged**, which is why the core takes `kv`. + +Errors are a **discriminated result, never a throw** — `processBookUpload` threw +`XRPCError` from a util, so a Hono route had to catch an HTTP-shaped exception +and translate it back. Each adapter owns its own status codes; the XRPC mapping +is `uploadErrorFor` in the router, the HTTP one is the `switch` in +`src/routes/library.tsx`. + +**Storage quota** is `SUM(personal_book.sizeBytes)` per user, derived rather than +counted: the quota itself bounds the row count (~700 rows at 2 GB over a ~3 MB +median epub), `idx_personal_book_user_size` (migration 023) makes it index-only, +and a derived total cannot drift. A counter would need a backfill, decrements in +both delete paths and a repair job — and `removeBookDir` is best-effort, so a +failed `rm` after a row delete would under-report forever while the disk filled. +**The quota counts book bytes only; stored covers are extra.** Measured on +production they add **~12%** on top (13.4 MiB of covers against 110.4 MiB of +books), so a user at a 2 GB quota occupies closer to 2.25 GB of disk — size the +volume accordingly rather than assuming the quota is the ceiling. +Over-quota is **413** (not 507, which proxies retry as a server fault; not 403, +which reads as auth) with `{error, code, usedBytes, quotaBytes}` — the iOS app +renders `payload.error` verbatim for any non-2xx, so already-installed builds +show the message with no update shipped. + +**Cover extraction is two-pass** (`bookMetadata/epub.ts`, `cbz.ts`). fflate's +`UnzipFileFilter` returning `false` still walks the central directory, so pass 1 +indexes every image's name and `originalSize` for free and pass 2 inflates +exactly the chosen one, gated on `MAX_COVER_BYTES`. Both parsers used to inflate +_every_ image in the archive to keep one — a 100 MB CBZ decompressed ~100 MB of +pages to keep page 1. ## Types & Constants @@ -487,10 +636,45 @@ SQLite-backed unstorage. Mounts: `search:` (in-memory LRU), `profile:`, `identit **XRPC list procedures**: `createList`, `updateList`, `deleteList`, `addToList`, `removeFromList`, `reorderList`. -**XRPC personal library queries**: `getPersonalLibrary`, `getPersonalBook`, `getSyncProgress`, `listSyncDocuments`. +**XRPC personal library queries**: `getPersonalLibrary` (params `limit`/`cursor`/`shelfId`/`q`/`sort`; output carries `storage`), `getPersonalBook`, `getPersonalBookFile`, `getPersonalBookCover`, `listPersonalShelves`, `getSyncProgress`, `listSyncDocuments`. **XRPC personal library procedures**: `uploadPersonalBook`, `deletePersonalBook`, `linkPersonalBook`, `unlinkPersonalBook`, `putSyncProgress`, `createPersonalShelf`, `updatePersonalShelf`, `deletePersonalShelf`, `addToPersonalShelf`, `removeFromPersonalShelf`. +**The personal library is fully reachable over XRPC, not just over `/opds`.** Parity map: + +| OPDS route | XRPC method | +| ----------------------------------- | -------------------------------------------------- | +| `GET /opds` (root nav + counts) | `listPersonalShelves` — the root call, one request | +| `GET /opds/all` | `getPersonalLibrary` | +| `GET /opds/shelves/:id` | `getPersonalLibrary?shelfId=` | +| `GET /opds/search/results` | `getPersonalLibrary?q=&sort=title` (same SQL) | +| `GET /opds/books/:hash/download` | `getPersonalBookFile` | +| `GET /opds/books/:hash/cover` | `getPersonalBookCover` | +| `GET /opds/search` (OpenSearch doc) | n/a — an XRPC client reads the lexicon instead | + +**Two methods declare non-JSON bodies**, which is what makes upload and download work at all: + +- `uploadPersonalBook` — `input.encoding` is an explicit ebook MIME list **plus + `application/octet-stream`**, and the filename is a required **query param** + (it replaced an `x-file-name` header that defaulted to `"unknown"`). Two + things to know before touching this: `@atcute/lex-cli` emits **no MIME + validator at all** when the encoding is exactly `*/*` (which is what the old + lexicon said), and `constructMimeValidator` is **exact-match** — `image/*` in + an encoding list is a literal, not a wildcard, so every type must be + enumerated. `octet-stream` is in the list deliberately: the iOS app sends + `type: mime || "application/octet-stream"`, mobile document pickers report it + for `.epub`, and `curl --data-binary` sends form-urlencoded. The declared + type is client-asserted and worthless as a control — **`detectFormat` + checking magic bytes against the filename's extension is the real gate**, and + `filename` is required because `.epub`/`.cbz`/`.fb2.zip` are all zip + containers that nothing else can tell apart. +- `getPersonalBookFile` / `getPersonalBookCover` — blob outputs, so the handler + returns a bare `Response` and owns every header (the `json()` helper is only + typed for lex outputs, and the router sets no Content-Type for you). Both are + in `ETAG_EXCLUDED_PREFIXES` and skipped by `compress()` in `src/app.ts`, by + **exact NSID path** rather than a `/xrpc/` prefix — the prefix would cost the + other ~35 JSON methods their 304s. + ## Scrapers (`src/scrapers/`) | File | Purpose | @@ -545,6 +729,63 @@ currently fails from the production host but not from a residential IP. **Key constraint**: `guardedRestore` wraps every OAuth restore in a 5s timeout + circuit breaker keyed by the authorization-server host. `getSessionAgent` only destroys sessions on terminal credential errors, never on timeouts. +### XRPC auth (`src/xrpc/auth.ts`) + +`/xrpc/*` accepts **two** credentials, resolved by `resolveXrpcAuth`: + +- the `sid` iron-session cookie (web + iOS, unchanged); +- an **atproto inter-service auth JWT** as `Authorization: Bearer ` + ([spec](https://atproto.com/specs/xrpc#inter-service-authentication-jwt)) — + the client calls `com.atproto.server.getServiceAuth` on its own PDS with + `aud` and `lxm`, and we verify with `ServiceJwtVerifier` from + `@atcute/xrpc-server/auth`. This is what makes the personal library usable + from a script or an e-reader rather than only from a browser session. + +Bearer wins when both are present. A method declares what it needs with +`auth: "identity" | "pdsWrite"` on its registration, and the wrapper in +`createXrpcRouter` **derives the `lxm` from the schema's own NSID**, so a +method's route and its token binding cannot drift apart. Handlers then read +`getAuth()` / `requireAgent()` instead of repeating a `getSessionAgent()` +preamble. + +- `identity` — we only need the DID. Every personal-library and sync method: + none of them touch the agent for anything but `.did` (progress bridging + writes `user_book` and queues a deferred PDS write via `sync_pending:`). +- `pdsWrite` — writes a record to the user's repo, so it needs a live OAuth + session. **Service auth can never satisfy this** — it proves key control, not + that we hold a grant. Only the six book-list procedures. + +Three things that are load-bearing and easy to get wrong: + +- **`acceptAudiences` is exact string `Array.includes`.** A bare DID does _not_ + match a `DID#fragment` audience, so both spellings are listed. The fragment + is there in advance of a PLC operation adding a `#bookhive_appview` service + entry; the live DID document has only `#atproto_pds`, so **PDS proxying via + `atproto-proxy` cannot work today** and clients must mint a token and POST to + us directly. +- **`XRPC_SERVICE_AUTH_MAX_AGE` defaults to 3600, not atcute's 300.** A PDS + mints up to an hour when `lxm` is set and most SDKs don't expose `exp`, so + the stricter default rejects ordinary tokens as `JwtTooOld`. The token's own + `exp` is still enforced separately. +- **`isKnownAccount` (`src/utils/account.ts`) is the gate**, and removing it is + a real hole: a valid token proves control of _an_ atproto identity, not that + it has ever used BookHive, so without it any DID on the network could open a + storage quota's worth of space on our disk. OPDS/KOSync get this implicitly + because their password derives from `COOKIE_SECRET`. + +`lexicons/auth.json` carries the `rpc` permission that lets a client mint these +tokens at all; **`GRANULAR_SCOPES` in `src/auth/client.ts` must move with it** +(it is the `USE_PERMISSION_SETS = false` fallback, and granting in only one +place silently drops it for whichever path is live). Note that adding `rpc` +permissions means **existing users must re-consent** before their PDS will +issue tokens for these methods. + +Replay protection (`src/xrpc/replay-store.ts`) ships **defaulted off**: it would +force a fresh `getServiceAuth` round-trip to the user's PDS on every call, to +close a `≤maxAge` window on a token already scoped to one `lxm` and one +audience. The insert is `ON CONFLICT DO NOTHING ... RETURNING` rather than an +unstorage get-then-set, which is not atomic across the four cluster processes. + ## Middleware (`src/middleware/`) Applied globally in `src/app.ts`. Key middleware: @@ -621,10 +862,51 @@ Keep selected/current states tinted and leave the solid fill to real actions. Notable deps: hono, kysely, zod 4, iron-session, unstorage + ocache, `@atcute/*`, `@takumi-rs/image-response` + React 19 (OG only), pino, `@hono/prometheus`, `@opentelemetry/*`, basecoat-css, envalid. +**`bunfig.toml` preloads `src/test/env-setup.ts`, and that is load-bearing.** +envalid freezes `env` at import, so `DB_PATH`/`LIBRARY_DIR` can only be set +before the import graph loads. Without it `DB_PATH` falls back to `":memory:"`, +`getLibraryDir()` resolves to **`./library` inside the checkout**, and any test +exercising an upload writes ebooks into the working tree. + +Related trap, since it cost real debugging time: **`mock.module("../env", …)` is +process-wide and permanent.** Returning a bare object from it replaces `env` for +every module loaded afterwards in the same run, so every field the mock doesn't +name reads back as `undefined`. `src/auth/session.test.ts` and +`token-refresh.test.ts` spread the real env for exactly this reason — the +failure shows up in an unrelated file that passes in isolation. + ## iOS App (`app/`) Separate Expo/React Native workspace — see `app/ARCHITECTURE.md`. Consumes personal-library and KOSync surfaces (XRPC `*PersonalBook`/`*PersonalShelf` methods, REST `/library/*`, `/settings/sync/*`, `POST /library/upload`). +## Third-party clients (service auth) + +What to tell someone who wants to script against the personal library: + +``` +# 1. Mint a token on YOUR OWN PDS, bound to one method and one audience. +# exp should be short; one token per call is what Bluesky's own PDS does. +GET https:///xrpc/com.atproto.server.getServiceAuth + ?aud=did:plc:enu2j5xjlqsjaylv3du4myh4 + &lxm=buzz.bookhive.uploadPersonalBook + &exp= + -> { "token": "..." } + +# 2. Call BookHive directly with it. `filename` is a required query param and +# the body is the raw file — no multipart wrapper. +POST https://bookhive.buzz/xrpc/buzz.bookhive.uploadPersonalBook?filename=Dune.epub + Authorization: Bearer + Content-Type: application/epub+zip + +``` + +Notes worth passing on: `lxm` is **required** (atcute's parser rejects a token +without it); the audience must match one of `acceptAudiences` **exactly**; +`atproto-proxy` will not work until the DID document gains an AppView service +entry; and the account must have signed in to BookHive at least once. The OAuth +client must have been granted the `rpc:buzz.bookhive.*` scopes, which means +**existing users need to re-consent** before their PDS will issue these tokens. + ## Workers, Logging & Observability | Path | Purpose | diff --git a/bun.lock b/bun.lock index e3f92fad..823718b7 100644 --- a/bun.lock +++ b/bun.lock @@ -21,6 +21,7 @@ "@atcute/atproto": "4.0.3", "@atcute/bluesky": "4.0.13", "@atcute/client": "5.1.1", + "@atcute/crypto": "2.4.2", "@atcute/identity-resolver": "2.0.1", "@atcute/identity-resolver-node": "2.0.1", "@atcute/jetstream": "2.0.1", diff --git a/bunfig.toml b/bunfig.toml index c9408fd0..c8cc8a55 100644 --- a/bunfig.toml +++ b/bunfig.toml @@ -4,3 +4,9 @@ # directories explicitly (`bun test src server`) so app/'s React Native suites, # which use the jest preset rather than bun's runner, stay out of the way. root = "." + +# Runs before src/env.ts is imported, which is the only moment DB_PATH and +# LIBRARY_DIR can still be set — envalid freezes `env` at import. Without it +# getLibraryDir() resolves to ./library inside the checkout and upload tests +# write ebooks into the repo. See src/test/env-setup.ts. +preload = ["./src/test/env-setup.ts"] diff --git a/lexicons/auth.json b/lexicons/auth.json index 63a40f39..10d1f629 100644 --- a/lexicons/auth.json +++ b/lexicons/auth.json @@ -5,7 +5,7 @@ "main": { "type": "permission-set", "title": "Full BookHive Functionality", - "detail": "Track books, write reviews and comments, and manage reading lists", + "detail": "Track books, write reviews and comments, manage reading lists, and use your personal ebook library from other apps", "permissions": [ { "type": "permission", @@ -18,6 +18,30 @@ "resource": "repo", "collection": ["buzz.bookhive.buzz"], "action": ["create", "update", "delete"] + }, + { + "type": "permission", + "resource": "rpc", + "aud": "*", + "lxm": [ + "buzz.bookhive.getPersonalLibrary", + "buzz.bookhive.getPersonalBook", + "buzz.bookhive.getPersonalBookFile", + "buzz.bookhive.getPersonalBookCover", + "buzz.bookhive.listPersonalShelves", + "buzz.bookhive.uploadPersonalBook", + "buzz.bookhive.deletePersonalBook", + "buzz.bookhive.linkPersonalBook", + "buzz.bookhive.unlinkPersonalBook", + "buzz.bookhive.createPersonalShelf", + "buzz.bookhive.updatePersonalShelf", + "buzz.bookhive.deletePersonalShelf", + "buzz.bookhive.addToPersonalShelf", + "buzz.bookhive.removeFromPersonalShelf", + "buzz.bookhive.getSyncProgress", + "buzz.bookhive.putSyncProgress", + "buzz.bookhive.listSyncDocuments" + ] } ] } diff --git a/lexicons/getPersonalBookCover.json b/lexicons/getPersonalBookCover.json new file mode 100644 index 00000000..7e2c8734 --- /dev/null +++ b/lexicons/getPersonalBookCover.json @@ -0,0 +1,36 @@ +{ + "lexicon": 1, + "id": "buzz.bookhive.getPersonalBookCover", + "defs": { + "main": { + "type": "query", + "description": "Cover image for a book in the authenticated user's personal library. Serves the cover extracted from the uploaded file when there is one; otherwise, if the book is linked to a BookHive catalog entry, responds 302 to the public image proxy for that entry. The XRPC equivalent of GET /opds/books/{hash}/cover.", + "parameters": { + "type": "params", + "required": ["contentHash"], + "properties": { + "contentHash": { + "type": "string", + "description": "Content hash identifying the book" + }, + "width": { + "type": "integer", + "description": "Requested width in pixels for the catalog-cover redirect. Ignored for locally stored covers, which are served at the size they were extracted at.", + "minimum": 32, + "maximum": 1024, + "default": 300 + } + } + }, + "output": { + "encoding": "image/jpeg,image/png,image/gif,image/webp" + }, + "errors": [ + { + "name": "NotFound", + "description": "No such book, or the book has neither a stored cover nor a linked catalog entry" + } + ] + } + } +} diff --git a/lexicons/getPersonalBookFile.json b/lexicons/getPersonalBookFile.json new file mode 100644 index 00000000..d2a6f704 --- /dev/null +++ b/lexicons/getPersonalBookFile.json @@ -0,0 +1,29 @@ +{ + "lexicon": 1, + "id": "buzz.bookhive.getPersonalBookFile", + "defs": { + "main": { + "type": "query", + "description": "Download the raw ebook file for a book in the authenticated user's personal library. Responds with the stored bytes, a strong ETag equal to the content hash, and a `Content-Disposition: attachment` filename. Honours `If-None-Match` with a 304, so an e-reader syncing on a schedule does not re-download every book. The XRPC equivalent of GET /opds/books/{hash}/download.", + "parameters": { + "type": "params", + "required": ["contentHash"], + "properties": { + "contentHash": { + "type": "string", + "description": "Content hash identifying the book, as returned by getPersonalLibrary" + } + } + }, + "output": { + "encoding": "application/epub+zip,application/x-mobipocket-ebook,application/x-fictionbook+xml,application/vnd.comicbook+zip,application/octet-stream" + }, + "errors": [ + { + "name": "NotFound", + "description": "No such book in the caller's library, or its file is missing from disk" + } + ] + } + } +} diff --git a/lexicons/getPersonalLibrary.json b/lexicons/getPersonalLibrary.json index a97eeaed..12257e1a 100644 --- a/lexicons/getPersonalLibrary.json +++ b/lexicons/getPersonalLibrary.json @@ -22,6 +22,18 @@ "shelfId": { "type": "integer", "description": "Filter by personal shelf ID" + }, + "q": { + "type": "string", + "description": "Case-insensitive substring match against title or authors. Mirrors the OPDS search feed.", + "minLength": 1, + "maxLength": 256 + }, + "sort": { + "type": "string", + "description": "Result ordering. `recent` is newest-added first (the default, matching the library page and the OPDS /all feed); `title` and `author` are ascending alphabetical, matching the OPDS search results feed. Not switched implicitly when `q` is set — pass it explicitly.", + "knownValues": ["recent", "title", "author"], + "default": "recent" } } }, @@ -45,11 +57,32 @@ "cursor": { "type": "string", "description": "Pagination cursor for the next page" + }, + "storage": { + "type": "ref", + "ref": "#storageView", + "description": "This user's storage usage against their quota" } } } } }, + "storageView": { + "type": "object", + "required": ["usedBytes", "quotaBytes"], + "properties": { + "usedBytes": { + "type": "integer", + "description": "Total bytes currently stored for this user", + "minimum": 0 + }, + "quotaBytes": { + "type": "integer", + "description": "Total bytes this user is allowed to store", + "minimum": 0 + } + } + }, "personalBookView": { "type": "object", "required": ["contentHash", "title", "format", "mime", "sizeBytes", "createdAt", "updatedAt"], @@ -88,7 +121,20 @@ }, "coverUrl": { "type": "string", - "description": "URL of the book cover image" + "description": "URL of the book cover image. When it points at the public catalog image proxy it needs no authentication; the `/library/covers/...` form is session-authenticated, so a client using service auth should use `hasLocalCover` and getPersonalBookCover instead." + }, + "hasLocalCover": { + "type": "boolean", + "description": "Whether a cover extracted from the uploaded file is stored. Fetch it with getPersonalBookCover, which works under any supported authentication." + }, + "filename": { + "type": "string", + "description": "Original uploaded file name. A sync client needs this to correlate the book with what is on the device." + }, + "description": { + "type": "string", + "description": "Synopsis from the linked BookHive catalog entry", + "maxLength": 5000 }, "createdAt": { "type": "string", diff --git a/lexicons/listPersonalShelves.json b/lexicons/listPersonalShelves.json new file mode 100644 index 00000000..7f8880a9 --- /dev/null +++ b/lexicons/listPersonalShelves.json @@ -0,0 +1,36 @@ +{ + "lexicon": 1, + "id": "buzz.bookhive.listPersonalShelves", + "defs": { + "main": { + "type": "query", + "description": "List the authenticated user's personal shelves with per-shelf book counts, plus library-wide totals and storage usage. This is the root call for a catalog client: it carries everything the OPDS root navigation feed renders, in one request. Unpaginated — `personal_shelf` is unique on (userDid, name) and these lists are small.", + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["shelves", "totalBooks"], + "properties": { + "shelves": { + "type": "array", + "items": { + "type": "ref", + "ref": "buzz.bookhive.createPersonalShelf#personalShelfView" + } + }, + "totalBooks": { + "type": "integer", + "description": "Books in the library across all shelves and unshelved", + "minimum": 0 + }, + "storage": { + "type": "ref", + "ref": "buzz.bookhive.getPersonalLibrary#storageView", + "description": "This user's storage usage against their quota" + } + } + } + } + } + } +} diff --git a/lexicons/uploadPersonalBook.json b/lexicons/uploadPersonalBook.json index 6739876f..dfeb6d42 100644 --- a/lexicons/uploadPersonalBook.json +++ b/lexicons/uploadPersonalBook.json @@ -4,9 +4,21 @@ "defs": { "main": { "type": "procedure", - "description": "Upload an ebook file to the personal library. Accepts EPUB, PDF, MOBI, FB2, CBZ.", + "description": "Upload an ebook file to the authenticated user's personal library. Accepts EPUB, MOBI/AZW/AZW3, FB2 (including .fb2.zip) and CBZ. The declared Content-Type is advisory only: the format is determined from the file's magic bytes checked against the `filename` extension, and a file that matches neither is rejected. `application/octet-stream` is accepted because mobile document pickers and plain curl uploads routinely send it.", + "parameters": { + "type": "params", + "required": ["filename"], + "properties": { + "filename": { + "type": "string", + "description": "Original file name including extension. Required: the extension is the only thing distinguishing the zip-container formats (.epub / .cbz / .fb2.zip) from each other, and it is the key used to match e-reader sync documents to this file.", + "minLength": 1, + "maxLength": 512 + } + } + }, "input": { - "encoding": "*/*" + "encoding": "application/epub+zip,application/x-mobipocket-ebook,application/vnd.amazon.ebook,application/vnd.amazon.mobi8-ebook,application/x-fictionbook+xml,application/vnd.comicbook+zip,application/x-cbz,application/zip,application/octet-stream" }, "output": { "encoding": "application/json", @@ -17,10 +29,42 @@ "book": { "type": "ref", "ref": "buzz.bookhive.getPersonalLibrary#personalBookView" + }, + "storageUsedBytes": { + "type": "integer", + "description": "Total bytes stored for this user after the upload", + "minimum": 0 + }, + "storageQuotaBytes": { + "type": "integer", + "description": "Total bytes this user is allowed to store", + "minimum": 0 } } } - } + }, + "errors": [ + { + "name": "UnsupportedFormat", + "description": "The bytes are not a recognised ebook, or do not match the filename's extension" + }, + { + "name": "AlreadyExists", + "description": "A book with this content hash is already in the library" + }, + { + "name": "TooLarge", + "description": "The file exceeds the per-file size limit" + }, + { + "name": "QuotaExceeded", + "description": "The upload would take the library over its total storage quota" + }, + { + "name": "Busy", + "description": "Too many uploads are being processed; retry shortly" + } + ] } } } diff --git a/package.json b/package.json index a31d3176..21f6b71a 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "@atcute/atproto": "4.0.3", "@atcute/bluesky": "4.0.13", "@atcute/client": "5.1.1", + "@atcute/crypto": "2.4.2", "@atcute/identity-resolver": "2.0.1", "@atcute/identity-resolver-node": "2.0.1", "@atcute/jetstream": "2.0.1", diff --git a/src/app.ts b/src/app.ts index 0a02bfef..ab2ef0eb 100644 --- a/src/app.ts +++ b/src/app.ts @@ -30,6 +30,20 @@ export function createApp({ startTime: serverStartTime, deps }: CreateAppOptions app.use(timing({ autoEnd: false })); + /** + * Routes that stream a stored ebook file. Kept out of both `compress()` and + * `etag()` — see the notes at each call site. + */ + const BOOK_DOWNLOAD_PREFIXES = [ + "/library/books/", + "/opds/books/", + "/xrpc/buzz.bookhive.getPersonalBookFile", + ]; + const isBookDownloadPath = (path: string) => + BOOK_DOWNLOAD_PREFIXES.some( + (prefix) => path === prefix || path.startsWith(prefix.endsWith("/") ? prefix : `${prefix}/`), + ); + if (env.isDevelopment) { app.use(prettyJSON()); } @@ -65,7 +79,16 @@ export function createApp({ startTime: serverStartTime, deps }: CreateAppOptions await next(); endTime(c, "compress"); }); - app.use(compress()); + const compressMiddleware = compress(); + app.use("*", async (c, next) => { + // Ebook downloads are already-compressed containers (EPUB and CBZ are ZIP, + // MOBI is its own packing), so gzipping them burns CPU for ~nothing. The + // one that would actually match hono's compressible-type regex is FB2 — + // `application/x-fictionbook+xml` hits the `+xml` branch — and compressing + // it drops the Content-Length a client is driving a progress bar from. + if (isBookDownloadPath(c.req.path)) return next(); + return compressMiddleware(c, next); + }); app.use(jsxRenderer()); @@ -122,7 +145,18 @@ export function createApp({ startTime: serverStartTime, deps }: CreateAppOptions // completes and no byte is ever flushed. That failure is severe and would // look like "import is broken" rather than "etag is misconfigured", so it is // worth being order-independent about. - const ETAG_EXCLUDED_PREFIXES = ["/library/books/", "/opds/books/", "/import"]; + // The two binary XRPC methods are listed by exact NSID, deliberately not as + // a `/xrpc/` prefix — that would cost the ~35 JSON methods their 304s. Both + // set their own ETag, and hono's etag() skips a response that already has + // one, so this is belt-and-braces: if a future edit drops that header the + // buffering regression above would otherwise come back silently. + const ETAG_EXCLUDED_PREFIXES = [ + "/library/books/", + "/opds/books/", + "/import", + "/xrpc/buzz.bookhive.getPersonalBookFile", + "/xrpc/buzz.bookhive.getPersonalBookCover", + ]; const isEtagExcluded = (path: string) => ETAG_EXCLUDED_PREFIXES.some( (prefix) => diff --git a/src/auth/client.ts b/src/auth/client.ts index 81ae465f..e3fc6549 100644 --- a/src/auth/client.ts +++ b/src/auth/client.ts @@ -13,8 +13,40 @@ export type LockFunction = (key: string, cb: () => Promise) => Promise; // When false, uses the granular per-resource scopes (works on all current PDS instances). const USE_PERMISSION_SETS = true; +/** + * The `rpc:buzz.bookhive.*` entries are what allow a client to mint an atproto + * service-auth token for our own XRPC methods via + * `com.atproto.server.getServiceAuth` — without them the user's PDS refuses. + * They must stay in lockstep with the `rpc` permission in `lexicons/auth.json`: + * this constant is the fallback used when `USE_PERMISSION_SETS` is false, and + * granting the permission in only one of the two places silently drops it for + * whichever path is live. + */ +const PERSONAL_LIBRARY_RPC_SCOPES = [ + "getPersonalLibrary", + "getPersonalBook", + "getPersonalBookFile", + "getPersonalBookCover", + "listPersonalShelves", + "uploadPersonalBook", + "deletePersonalBook", + "linkPersonalBook", + "unlinkPersonalBook", + "createPersonalShelf", + "updatePersonalShelf", + "deletePersonalShelf", + "addToPersonalShelf", + "removeFromPersonalShelf", + "getSyncProgress", + "putSyncProgress", + "listSyncDocuments", +] + .map((method) => `rpc:buzz.bookhive.${method}?aud=*`) + .join(" "); + const GRANULAR_SCOPES = - "atproto blob:*/* repo:buzz.bookhive.book?action=create&action=update&action=delete repo:buzz.bookhive.buzz?action=create&action=update&action=delete repo:app.bsky.graph.follow?action=create&action=delete repo:social.popfeed.feed.list?action=create&action=update&action=delete repo:social.popfeed.feed.listItem?action=create&action=update&action=delete rpc:app.bsky.graph.getFollows?aud=* rpc:app.bsky.actor.getProfile?aud=* rpc:app.bsky.actor.getProfiles?aud=*"; + "atproto blob:*/* repo:buzz.bookhive.book?action=create&action=update&action=delete repo:buzz.bookhive.buzz?action=create&action=update&action=delete repo:app.bsky.graph.follow?action=create&action=delete repo:social.popfeed.feed.list?action=create&action=update&action=delete repo:social.popfeed.feed.listItem?action=create&action=update&action=delete rpc:app.bsky.graph.getFollows?aud=* rpc:app.bsky.actor.getProfile?aud=* rpc:app.bsky.actor.getProfiles?aud=* " + + PERSONAL_LIBRARY_RPC_SCOPES; // Permission set can only cover buzz.bookhive.* namespace (spec namespace authority rule). // blob, app.bsky.*, and social.popfeed.* must remain as granular scopes. diff --git a/src/auth/router.tsx b/src/auth/router.tsx index 772b6d9f..c1097609 100644 --- a/src/auth/router.tsx +++ b/src/auth/router.tsx @@ -22,6 +22,7 @@ import { uploadBlob, } from "../pds/client"; import { generateInitialsAvatar } from "../utils/generateInitialsAvatar"; +import { markAccount } from "../utils/account"; // Helper function to get consistent session configuration export function getSessionConfig(): SessionOptions { @@ -82,6 +83,12 @@ export function loginRouter( clientSession.did = session.did as string; await clientSession.save(); + // Marks this DID as a BookHive account, which is the gate service auth + // checks — a valid inter-service JWT proves identity, not that the + // identity has ever used us. Best-effort: a failure here would only cost + // the (indexed, one-off) backfill probe in isKnownAccount. + void markAccount(c.get("ctx").kv, session.did as string).catch(() => {}); + const agent = sessionClientFromOAuthSession(session); // Pre-warm the in-memory session cache so the first request after login diff --git a/src/auth/session.test.ts b/src/auth/session.test.ts index 34b28bef..0eeab58a 100644 --- a/src/auth/session.test.ts +++ b/src/auth/session.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, mock } from "bun:test"; import { getIronSession } from "iron-session"; +import { env as realEnv } from "../env"; // Mock iron-session const mockGetIronSession = mock(); @@ -7,11 +8,16 @@ void mock.module("iron-session", () => ({ getIronSession: mockGetIronSession, })); -// Mock environment +// Override one field of the environment, keeping the rest. +// +// `mock.module` is process-wide and permanent, so returning a bare object here +// replaces `env` for *every* module loaded afterwards in the same `bun test` +// run — every other field reads back as undefined. That is a landmine for any +// module that reads env lazily inside a request (`getLibraryDir()` reading +// `DB_PATH`, for one): the owning test passes in isolation and an unrelated +// file fails in the full suite. Spread the real env so only COOKIE_SECRET moves. void mock.module("../env", () => ({ - env: { - COOKIE_SECRET: "test-secret-key-for-testing-purposes-only", - }, + env: { ...realEnv, COOKIE_SECRET: "test-secret-key-for-testing-purposes-only" }, })); describe("Auth Session TTL Logic", () => { diff --git a/src/auth/token-refresh.test.ts b/src/auth/token-refresh.test.ts index 286799a0..5e9d4076 100644 --- a/src/auth/token-refresh.test.ts +++ b/src/auth/token-refresh.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, mock, beforeEach } from "bun:test"; +import { env as realEnv } from "../env"; // Mock the OAuth client and session const mockOAuthSession = { @@ -16,11 +17,11 @@ const mockSession = { destroy: mock(), }; -// Mock environment +// Override one field of the environment, keeping the rest — see the same note +// in session.test.ts. `mock.module` is process-wide, so a bare object here +// blanks every other env field for the remainder of the run. void mock.module("../env", () => ({ - env: { - COOKIE_SECRET: "test-secret-key-for-testing-purposes-only", - }, + env: { ...realEnv, COOKIE_SECRET: "test-secret-key-for-testing-purposes-only" }, })); describe("Token Refresh Logic", () => { diff --git a/src/bsky/id-resolver.ts b/src/bsky/id-resolver.ts index 319f925c..f47f77b4 100644 --- a/src/bsky/id-resolver.ts +++ b/src/bsky/id-resolver.ts @@ -1,6 +1,6 @@ import type { Storage } from "unstorage"; import type { ActorIdentifier } from "@atcute/lexicons/syntax"; -import type { ActorResolver } from "@atcute/identity-resolver"; +import type { ActorResolver, DidDocumentResolver } from "@atcute/identity-resolver"; import { CompositeDidDocumentResolver, CompositeHandleResolver, @@ -13,6 +13,26 @@ import { NodeDnsHandleResolver } from "@atcute/identity-resolver-node"; import { readThroughCache } from "../utils/readThroughCache"; +let sharedDidDocumentResolver: DidDocumentResolver | undefined; + +/** + * The process-wide DID document resolver. + * + * Memoised rather than constructed per caller so the OAuth client, the handle + * resolvers and the service-auth JWT verifier all share one instance — and + * therefore one DID-document cache. The verifier re-resolves with + * `noCache: true` when a signature fails, to recover from key rotation; + * `CompositeDidDocumentResolver` forwards that through, so sharing is safe. + */ +export function getDidDocumentResolver(): DidDocumentResolver { + return (sharedDidDocumentResolver ??= new CompositeDidDocumentResolver({ + methods: { + plc: new PlcDidDocumentResolver(), + web: new WebDidDocumentResolver(), + }, + })); +} + /** Create ActorResolver for OAuth (handle/DID resolution). */ export function createActorResolver(): ActorResolver { const handleResolver = new CompositeHandleResolver({ @@ -21,15 +41,9 @@ export function createActorResolver(): ActorResolver { http: new WellKnownHandleResolver(), }, }); - const didDocumentResolver = new CompositeDidDocumentResolver({ - methods: { - plc: new PlcDidDocumentResolver(), - web: new WebDidDocumentResolver(), - }, - }); return new LocalActorResolver({ handleResolver, - didDocumentResolver, + didDocumentResolver: getDidDocumentResolver(), }); } diff --git a/src/bsky/lexicon/generated/index.ts b/src/bsky/lexicon/generated/index.ts index 98b0c2a7..b64392a2 100644 --- a/src/bsky/lexicon/generated/index.ts +++ b/src/bsky/lexicon/generated/index.ts @@ -17,6 +17,8 @@ export * as BuzzBookhiveGetFeed from "./types/buzz/bookhive/getFeed.js"; export * as BuzzBookhiveGetLanguages from "./types/buzz/bookhive/getLanguages.js"; export * as BuzzBookhiveGetList from "./types/buzz/bookhive/getList.js"; export * as BuzzBookhiveGetPersonalBook from "./types/buzz/bookhive/getPersonalBook.js"; +export * as BuzzBookhiveGetPersonalBookCover from "./types/buzz/bookhive/getPersonalBookCover.js"; +export * as BuzzBookhiveGetPersonalBookFile from "./types/buzz/bookhive/getPersonalBookFile.js"; export * as BuzzBookhiveGetPersonalLibrary from "./types/buzz/bookhive/getPersonalLibrary.js"; export * as BuzzBookhiveGetProfile from "./types/buzz/bookhive/getProfile.js"; export * as BuzzBookhiveGetReadingStats from "./types/buzz/bookhive/getReadingStats.js"; @@ -25,6 +27,7 @@ export * as BuzzBookhiveGetUserLists from "./types/buzz/bookhive/getUserLists.js export * as BuzzBookhiveHiveBook from "./types/buzz/bookhive/hiveBook.js"; export * as BuzzBookhiveLinkPersonalBook from "./types/buzz/bookhive/linkPersonalBook.js"; export * as BuzzBookhiveListGenres from "./types/buzz/bookhive/listGenres.js"; +export * as BuzzBookhiveListPersonalShelves from "./types/buzz/bookhive/listPersonalShelves.js"; export * as BuzzBookhiveListSyncDocuments from "./types/buzz/bookhive/listSyncDocuments.js"; export * as BuzzBookhivePutSyncProgress from "./types/buzz/bookhive/putSyncProgress.js"; export * as BuzzBookhiveRemoveFromList from "./types/buzz/bookhive/removeFromList.js"; diff --git a/src/bsky/lexicon/generated/types/buzz/bookhive/getPersonalBookCover.ts b/src/bsky/lexicon/generated/types/buzz/bookhive/getPersonalBookCover.ts new file mode 100644 index 00000000..a143009c --- /dev/null +++ b/src/bsky/lexicon/generated/types/buzz/bookhive/getPersonalBookCover.ts @@ -0,0 +1,40 @@ +import type {} from "@atcute/lexicons"; +import * as v from "@atcute/lexicons/validations"; +import type {} from "@atcute/lexicons/ambient"; + +const _mainSchema = /*#__PURE__*/ v.query("buzz.bookhive.getPersonalBookCover", { + params: /*#__PURE__*/ v.object({ + /** + * Content hash identifying the book + */ + contentHash: /*#__PURE__*/ v.string(), + /** + * Requested width in pixels for the catalog-cover redirect. Ignored for locally stored covers, which are served at the size they were extracted at. + * @minimum 32 + * @maximum 1024 + * @default 300 + */ + width: /*#__PURE__*/ v.optional( + /*#__PURE__*/ v.constrain(/*#__PURE__*/ v.integer(), [ + /*#__PURE__*/ v.integerRange(32, 1024), + ]), + 300, + ), + }), + output: { + type: "blob", + encoding: ["image/jpeg", "image/png", "image/gif", "image/webp"], + }, +}); +type main$schematype = typeof _mainSchema; + +export interface mainSchema extends main$schematype {} +export const mainSchema = _mainSchema as mainSchema; + +export interface $params extends v.InferInput {} +export type $output = v.InferXRPCBodyInput; +declare module "@atcute/lexicons/ambient" { + interface XRPCQueries { + "buzz.bookhive.getPersonalBookCover": mainSchema; + } +} diff --git a/src/bsky/lexicon/generated/types/buzz/bookhive/getPersonalBookFile.ts b/src/bsky/lexicon/generated/types/buzz/bookhive/getPersonalBookFile.ts new file mode 100644 index 00000000..e6fd33c1 --- /dev/null +++ b/src/bsky/lexicon/generated/types/buzz/bookhive/getPersonalBookFile.ts @@ -0,0 +1,34 @@ +import type {} from "@atcute/lexicons"; +import * as v from "@atcute/lexicons/validations"; +import type {} from "@atcute/lexicons/ambient"; + +const _mainSchema = /*#__PURE__*/ v.query("buzz.bookhive.getPersonalBookFile", { + params: /*#__PURE__*/ v.object({ + /** + * Content hash identifying the book, as returned by getPersonalLibrary + */ + contentHash: /*#__PURE__*/ v.string(), + }), + output: { + type: "blob", + encoding: [ + "application/epub+zip", + "application/x-mobipocket-ebook", + "application/x-fictionbook+xml", + "application/vnd.comicbook+zip", + "application/octet-stream", + ], + }, +}); +type main$schematype = typeof _mainSchema; + +export interface mainSchema extends main$schematype {} +export const mainSchema = _mainSchema as mainSchema; + +export interface $params extends v.InferInput {} +export type $output = v.InferXRPCBodyInput; +declare module "@atcute/lexicons/ambient" { + interface XRPCQueries { + "buzz.bookhive.getPersonalBookFile": mainSchema; + } +} diff --git a/src/bsky/lexicon/generated/types/buzz/bookhive/getPersonalLibrary.ts b/src/bsky/lexicon/generated/types/buzz/bookhive/getPersonalLibrary.ts index 4937b5ca..95064605 100644 --- a/src/bsky/lexicon/generated/types/buzz/bookhive/getPersonalLibrary.ts +++ b/src/bsky/lexicon/generated/types/buzz/bookhive/getPersonalLibrary.ts @@ -18,10 +18,26 @@ const _mainSchema = /*#__PURE__*/ v.query("buzz.bookhive.getPersonalLibrary", { /*#__PURE__*/ v.constrain(/*#__PURE__*/ v.integer(), [/*#__PURE__*/ v.integerRange(1, 100)]), 24, ), + /** + * Case-insensitive substring match against title or authors. Mirrors the OPDS search feed. + * @minLength 1 + * @maxLength 256 + */ + q: /*#__PURE__*/ v.optional( + /*#__PURE__*/ v.constrain(/*#__PURE__*/ v.string(), [/*#__PURE__*/ v.stringLength(1, 256)]), + ), /** * Filter by personal shelf ID */ shelfId: /*#__PURE__*/ v.optional(/*#__PURE__*/ v.integer()), + /** + * Result ordering. `recent` is newest-added first (the default, matching the library page and the OPDS /all feed); `title` and `author` are ascending alphabetical, matching the OPDS search results feed. Not switched implicitly when `q` is set — pass it explicitly. + * @default "recent" + */ + sort: /*#__PURE__*/ v.optional( + /*#__PURE__*/ v.string<"author" | "recent" | "title" | (string & {})>(), + "recent", + ), }), output: { type: "lex", @@ -33,6 +49,12 @@ const _mainSchema = /*#__PURE__*/ v.query("buzz.bookhive.getPersonalLibrary", { * Pagination cursor for the next page */ cursor: /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), + /** + * This user's storage usage against their quota + */ + get storage() { + return /*#__PURE__*/ v.optional(storageViewSchema); + }, /** * Total number of books matching the query, across all pages */ @@ -53,17 +75,32 @@ const _personalBookViewSchema = /*#__PURE__*/ v.object({ */ contentHash: /*#__PURE__*/ v.string(), /** - * URL of the book cover image + * URL of the book cover image. When it points at the public catalog image proxy it needs no authentication; the `/library/covers/...` form is session-authenticated, so a client using service auth should use `hasLocalCover` and getPersonalBookCover instead. */ coverUrl: /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), /** * When the book was added to the library */ createdAt: /*#__PURE__*/ v.datetimeString(), + /** + * Synopsis from the linked BookHive catalog entry + * @maxLength 5000 + */ + description: /*#__PURE__*/ v.optional( + /*#__PURE__*/ v.constrain(/*#__PURE__*/ v.string(), [/*#__PURE__*/ v.stringLength(0, 5000)]), + ), + /** + * Original uploaded file name. A sync client needs this to correlate the book with what is on the device. + */ + filename: /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), /** * File format (e.g. epub, pdf, mobi, fb2, cbz) */ format: /*#__PURE__*/ v.string(), + /** + * Whether a cover extracted from the uploaded file is stored. Fetch it with getPersonalBookCover, which works under any supported authentication. + */ + hasLocalCover: /*#__PURE__*/ v.optional(/*#__PURE__*/ v.boolean()), /** * Linked BookHive catalog entry ID */ @@ -99,6 +136,21 @@ const _personalBookViewSchema = /*#__PURE__*/ v.object({ */ updatedAt: /*#__PURE__*/ v.datetimeString(), }); +const _storageViewSchema = /*#__PURE__*/ v.object({ + $type: /*#__PURE__*/ v.optional( + /*#__PURE__*/ v.literal("buzz.bookhive.getPersonalLibrary#storageView"), + ), + /** + * Total bytes this user is allowed to store + * @minimum 0 + */ + quotaBytes: /*#__PURE__*/ v.integer(), + /** + * Total bytes currently stored for this user + * @minimum 0 + */ + usedBytes: /*#__PURE__*/ v.integer(), +}); const _syncProgressViewSchema = /*#__PURE__*/ v.object({ $type: /*#__PURE__*/ v.optional( /*#__PURE__*/ v.literal("buzz.bookhive.getPersonalLibrary#syncProgressView"), @@ -118,19 +170,25 @@ const _syncProgressViewSchema = /*#__PURE__*/ v.object({ }); type main$schematype = typeof _mainSchema; type personalBookView$schematype = typeof _personalBookViewSchema; +type storageView$schematype = typeof _storageViewSchema; type syncProgressView$schematype = typeof _syncProgressViewSchema; export interface mainSchema extends main$schematype {} export interface personalBookViewSchema extends personalBookView$schematype {} +export interface storageViewSchema extends storageView$schematype {} + export interface syncProgressViewSchema extends syncProgressView$schematype {} export const mainSchema = _mainSchema as mainSchema; export const personalBookViewSchema = _personalBookViewSchema as personalBookViewSchema; +export const storageViewSchema = _storageViewSchema as storageViewSchema; export const syncProgressViewSchema = _syncProgressViewSchema as syncProgressViewSchema; export interface PersonalBookView extends v.InferInput {} +export interface StorageView extends v.InferInput {} + export interface SyncProgressView extends v.InferInput {} export interface $params extends v.InferInput {} diff --git a/src/bsky/lexicon/generated/types/buzz/bookhive/listPersonalShelves.ts b/src/bsky/lexicon/generated/types/buzz/bookhive/listPersonalShelves.ts new file mode 100644 index 00000000..ff35e1c5 --- /dev/null +++ b/src/bsky/lexicon/generated/types/buzz/bookhive/listPersonalShelves.ts @@ -0,0 +1,41 @@ +import type {} from "@atcute/lexicons"; +import * as v from "@atcute/lexicons/validations"; +import type {} from "@atcute/lexicons/ambient"; +import * as BuzzBookhiveCreatePersonalShelf from "./createPersonalShelf.js"; +import * as BuzzBookhiveGetPersonalLibrary from "./getPersonalLibrary.js"; + +const _mainSchema = /*#__PURE__*/ v.query("buzz.bookhive.listPersonalShelves", { + params: null, + output: { + type: "lex", + schema: /*#__PURE__*/ v.object({ + get shelves() { + return /*#__PURE__*/ v.array(BuzzBookhiveCreatePersonalShelf.personalShelfViewSchema); + }, + /** + * This user's storage usage against their quota + */ + get storage() { + return /*#__PURE__*/ v.optional(BuzzBookhiveGetPersonalLibrary.storageViewSchema); + }, + /** + * Books in the library across all shelves and unshelved + * @minimum 0 + */ + totalBooks: /*#__PURE__*/ v.integer(), + }), + }, +}); +type main$schematype = typeof _mainSchema; + +export interface mainSchema extends main$schematype {} +export const mainSchema = _mainSchema as mainSchema; + +export interface $params {} + +export interface $output extends v.InferXRPCBodyInput {} +declare module "@atcute/lexicons/ambient" { + interface XRPCQueries { + "buzz.bookhive.listPersonalShelves": mainSchema; + } +} diff --git a/src/bsky/lexicon/generated/types/buzz/bookhive/uploadPersonalBook.ts b/src/bsky/lexicon/generated/types/buzz/bookhive/uploadPersonalBook.ts index f3cc1e1a..0dbbccc0 100644 --- a/src/bsky/lexicon/generated/types/buzz/bookhive/uploadPersonalBook.ts +++ b/src/bsky/lexicon/generated/types/buzz/bookhive/uploadPersonalBook.ts @@ -4,14 +4,46 @@ import type {} from "@atcute/lexicons/ambient"; import * as BuzzBookhiveGetPersonalLibrary from "./getPersonalLibrary.js"; const _mainSchema = /*#__PURE__*/ v.procedure("buzz.bookhive.uploadPersonalBook", { - params: null, - input: { type: "blob" }, + params: /*#__PURE__*/ v.object({ + /** + * Original file name including extension. Required: the extension is the only thing distinguishing the zip-container formats (.epub / .cbz / .fb2.zip) from each other, and it is the key used to match e-reader sync documents to this file. + * @minLength 1 + * @maxLength 512 + */ + filename: /*#__PURE__*/ v.constrain(/*#__PURE__*/ v.string(), [ + /*#__PURE__*/ v.stringLength(1, 512), + ]), + }), + input: { + type: "blob", + encoding: [ + "application/epub+zip", + "application/x-mobipocket-ebook", + "application/vnd.amazon.ebook", + "application/vnd.amazon.mobi8-ebook", + "application/x-fictionbook+xml", + "application/vnd.comicbook+zip", + "application/x-cbz", + "application/zip", + "application/octet-stream", + ], + }, output: { type: "lex", schema: /*#__PURE__*/ v.object({ get book() { return BuzzBookhiveGetPersonalLibrary.personalBookViewSchema; }, + /** + * Total bytes this user is allowed to store + * @minimum 0 + */ + storageQuotaBytes: /*#__PURE__*/ v.optional(/*#__PURE__*/ v.integer()), + /** + * Total bytes stored for this user after the upload + * @minimum 0 + */ + storageUsedBytes: /*#__PURE__*/ v.optional(/*#__PURE__*/ v.integer()), }), }, }); @@ -20,7 +52,7 @@ type main$schematype = typeof _mainSchema; export interface mainSchema extends main$schematype {} export const mainSchema = _mainSchema as mainSchema; -export interface $params {} +export interface $params extends v.InferInput {} export type $input = v.InferXRPCBodyInput; export interface $output extends v.InferXRPCBodyInput {} diff --git a/src/client/components/LibraryManager.tsx b/src/client/components/LibraryManager.tsx index e97ae045..d6bf5e32 100644 --- a/src/client/components/LibraryManager.tsx +++ b/src/client/components/LibraryManager.tsx @@ -37,6 +37,51 @@ const getJson = (url: string): Promise => fetch(url, { cache: "no-stor * triaged above the grid, or parked in "Also tracking" below it once the user * has linked or dismissed them. */ +function formatBytes(bytes: number): string { + if (bytes >= 1024 ** 3) return `${(bytes / 1024 ** 3).toFixed(1)} GB`; + if (bytes >= 1024 ** 2) return `${Math.round(bytes / 1024 ** 2)} MB`; + return `${Math.round(bytes / 1024)} KB`; +} + +/** + * Storage against quota. Deliberately quiet until it matters: below 60% this is + * just a line of text, and the bar only appears once the number is worth acting + * on. Better for the user to see it filling up here than to push 100 MB up the + * wire and get a 413. + */ +const StorageMeter: FC<{ storage: { usedBytes: number; quotaBytes: number } | null }> = ({ + storage, +}) => { + if (!storage || storage.quotaBytes <= 0) return null; + const ratio = Math.min(1, storage.usedBytes / storage.quotaBytes); + const pct = Math.round(ratio * 100); + const full = ratio >= 1; + + return ( +
+ + {formatBytes(storage.usedBytes)} of {formatBytes(storage.quotaBytes)} used + + {ratio >= 0.6 && ( + + + + )} + {full && Library full — delete a book to upload more.} +
+ ); +}; + export const LibraryManager: FC = () => { const [books, setBooks] = useState(null); const [cursor, setCursor] = useState(undefined); @@ -48,6 +93,7 @@ export const LibraryManager: FC = () => { const [shelves, setShelves] = useState([]); const [shelvesLoaded, setShelvesLoaded] = useState(false); const [totalBooks, setTotalBooks] = useState(0); + const [storage, setStorage] = useState<{ usedBytes: number; quotaBytes: number } | null>(null); const [activeShelfId, setActiveShelfId] = useState(null); const [linkTarget, setLinkTarget] = useState(null); @@ -72,11 +118,21 @@ export const LibraryManager: FC = () => { if (!r.ok) throw new Error("Failed"); return r.json(); }) - .then((d: { books: PersonalBook[]; total?: number; cursor?: string }) => { - setBooks(d.books); - setCursor(d.cursor); - if (shelfId === null) setTotalBooks(d.total ?? d.books.length); - }) + .then( + (d: { + books: PersonalBook[]; + total?: number; + cursor?: string; + storage?: { usedBytes: number; quotaBytes: number }; + }) => { + setBooks(d.books); + setCursor(d.cursor); + if (shelfId === null) setTotalBooks(d.total ?? d.books.length); + // Rides along on the list every client already refetches after each + // mutation, so the usage bar stays current with no extra request. + if (d.storage) setStorage(d.storage); + }, + ) .catch(() => setError(true)) .finally(() => setRefreshing(false)); }, []); @@ -393,6 +449,8 @@ export const LibraryManager: FC = () => { /> )} + + {error && (

Could not load your library. Try reloading the page. diff --git a/src/context.ts b/src/context.ts index 041afac9..fecbfb0e 100644 --- a/src/context.ts +++ b/src/context.ts @@ -27,8 +27,15 @@ import { createBidirectionalResolverAtcute, createCachingBaseIdResolver, createCachingBidirectionalResolver, + getDidDocumentResolver, } from "./bsky/id-resolver"; import type { BidirectionalResolver } from "./bsky/id-resolver"; +import { ServiceJwtVerifier, type ReplayStore } from "@atcute/xrpc-server/auth"; +import type { AtprotoAudience } from "@atcute/lexicons/syntax"; +import { BOOKHIVE_DID } from "./constants"; +import { isKnownAccount } from "./utils/account"; +import { sweepStaleUploads } from "./utils/uploadPersonalBook"; +import { createKvReplayStore, ensureReplayTable, sweepReplayStore } from "./xrpc/replay-store"; import type { Database } from "./db"; import { createDb, migrateToLatest } from "./db"; import { env } from "./env"; @@ -72,6 +79,10 @@ export type AppContext = { getProfile: () => Promise; /** Service account agent for @bookhive.buzz ATProto writes. Null if env vars not set. */ serviceAccountAgent: SessionClient | null; + /** Verifies atproto service-auth JWTs on /xrpc/*. Null when disabled. */ + serviceJwtVerifier: ServiceJwtVerifier | null; + /** Gate on service auth: has this DID ever used BookHive? See utils/account.ts. */ + isKnownAccount: (did: string) => Promise; /** Add fields to the one wide event logged per request (observability). */ addWideEventContext: AddWideEventContext; }; @@ -117,6 +128,8 @@ export type AppDeps = { ingester: Ingester; resolver: BidirectionalResolver; serviceAccountAgent: SessionClient | null; + /** Verifies atproto service-auth JWTs on /xrpc/*. Null when disabled. */ + serviceJwtVerifier: ServiceJwtVerifier | null; /** Stops the primary worker's enrichment drain loop; no-op elsewhere. */ stopEnrichmentDrain: () => void; }; @@ -224,6 +237,9 @@ export async function createAppDeps(): Promise { kv.mount("sync_pending:", sqliteKv({ table: "sync_pending", db: kvDb })); // Per-user KOSync token rotation counter (see src/middleware/sync-auth.ts). kv.mount("sync_token:", sqliteKv({ table: "sync_token", db: kvDb })); + // "This DID has used BookHive" marker, the gate on service auth. Permanent — + // see src/utils/account.ts for why it exists and why it never expires. + kv.mount("account:", sqliteKv({ table: "account", db: kvDb })); // Anonymous full-page HTML cache (see src/middleware/anon-page-cache.ts). kv.mount("page:", sqliteKv({ table: "page_cache", db: kvDb })); if (isPrimaryWorker) { @@ -304,6 +320,57 @@ export async function createAppDeps(): Promise { // scrape per search result (the 2026-08-01 OOM). const stopEnrichmentDrain = isPrimaryWorker ? startEnrichmentDrain({ db, logger }) : () => {}; + // Accepts atproto inter-service auth on /xrpc/*, which is what lets a client + // that is not a browser (a script, an e-reader, another app) use the personal + // library. Null disables the Bearer path entirely; the cookie path is + // unaffected either way. + let serviceJwtVerifier: ServiceJwtVerifier | null = null; + if (env.XRPC_SERVICE_AUTH) { + let replayStore: ReplayStore | undefined; + if (env.XRPC_SERVICE_AUTH_REPLAY) { + await ensureReplayTable(kvDb); + replayStore = createKvReplayStore(kvDb); + } + serviceJwtVerifier = new ServiceJwtVerifier({ + // atcute compares these with exact string equality, so a bare DID does + // *not* match a `#fragment` audience — both spellings have to be listed. + // The fragment form is here in advance of a PLC operation adding a + // `#bookhive_appview` service entry to the DID document; once that lands, + // clients that switch to PDS proxying keep working with no server change. + acceptAudiences: serviceAuthAudiences(), + resolver: getDidDocumentResolver(), + // atcute defaults to 300s, but a PDS mints up to 3600s when `lxm` is set + // and most client SDKs don't expose `exp`. `exp` itself is still enforced + // independently, so widening this only accepts tokens the issuing PDS + // already considered valid. + maxAge: env.XRPC_SERVICE_AUTH_MAX_AGE, + ...(replayStore ? { replayStore } : {}), + }); + } + + if (isPrimaryWorker) { + // Clear `.part` files left by a process that died between an upload's write + // and its rename. Nothing else removes them, and each is up to 100 MB. + void sweepStaleUploads().then( + (removed) => { + if (removed > 0) logger.info({ removed }, "swept stale upload temp files"); + }, + (err: unknown) => logger.warn({ err }, "stale upload sweep failed"), + ); + } + + if (isPrimaryWorker && env.XRPC_SERVICE_AUTH_REPLAY) { + const replaySweep = setInterval( + () => { + void sweepReplayStore(kvDb).catch((err: unknown) => { + logger.warn({ err }, "service-auth replay sweep failed"); + }); + }, + 15 * 60 * 1000, + ); + replaySweep.unref?.(); + } + return { db, kv, @@ -313,10 +380,23 @@ export async function createAppDeps(): Promise { ingester, resolver, serviceAccountAgent, + serviceJwtVerifier, stopEnrichmentDrain, }; } +/** + * The `aud` values this deployment answers for. Overridable so a staging + * deployment can run under its own DID without a code change. + */ +function serviceAuthAudiences(): (Did | AtprotoAudience)[] { + const override = env.XRPC_SERVICE_AUTH_AUDIENCES.split(",") + .map((s) => s.trim()) + .filter(Boolean); + if (override.length > 0) return override as (Did | AtprotoAudience)[]; + return [BOOKHIVE_DID, `${BOOKHIVE_DID}#bookhive_appview`] as (Did | AtprotoAudience)[]; +} + /** Optional timing callbacks for server-timing breakdown (session_iron, session_restore, session_save). */ export type SessionTiming = { start: (name: string) => void; @@ -544,6 +624,9 @@ export function createContextMiddleware(deps: AppDeps) { addWideEventContext(context: Record) { Object.assign(c.get("wideEventBag"), context); }, + isKnownAccount(did: string): Promise { + return isKnownAccount({ db: deps.db, kv: deps.kv }, did); + }, getSessionDid(): Promise { return didLazy.value; }, diff --git a/src/db.ts b/src/db.ts index 33c2683a..231e2438 100644 --- a/src/db.ts +++ b/src/db.ts @@ -20,6 +20,7 @@ import type { UserFollow, } from "./types"; import { deriveBookIdentifiers } from "./utils/bookIdentifiers.js"; +import { filenameKey, koreaderFilenameHash } from "./utils/filenameMatching.js"; // Types export type DatabaseSchema = { @@ -908,6 +909,98 @@ migrations["021"] = { }, }; +migrations["022"] = { + async up(db: Kysely) { + // Filename-derived identity for e-reader documents. See + // src/utils/filenameMatching.ts for what each value is and why the first is + // exact while the second is not: + // + // - `filenameHash` is md5(basename) — literally the `document` id a KOSync + // client sends when its checksum method is FILENAME instead of BINARY. + // Without it, every user on that setting has a library where no uploaded + // file ever lines up with its synced progress, because the id they send + // is not a content hash at all. + // - `filenameKey` is a normalized, extension-less name, so a file survives + // the calibre conversion (.epub -> .azw3) that broke the content hash in + // the first place — which is the reason those users switched. + await db.schema.alterTable("personal_book").addColumn("filenameHash", "text").execute(); + await db.schema.alterTable("personal_book").addColumn("filenameKey", "text").execute(); + await db.schema.alterTable("sync_document").addColumn("filenameKey", "text").execute(); + + await sql`CREATE INDEX idx_personal_book_user_filename_hash ON personal_book(userDid, filenameHash)`.execute( + db, + ); + await sql`CREATE INDEX idx_personal_book_user_filename_key ON personal_book(userDid, filenameKey)`.execute( + db, + ); + await sql`CREATE INDEX idx_sync_document_user_filename_key ON sync_document(userDid, filenameKey)`.execute( + db, + ); + + // Backfill in JS: SQLite has no md5, and the normalization is Unicode-aware. + // Both tables hold one row per user per book, so this is small. + const books = ( + await sql<{ + id: number; + filename: string | null; + }>`SELECT id, filename FROM personal_book`.execute(db) + ).rows; + for (const row of books) { + const hash = koreaderFilenameHash(row.filename); + const key = filenameKey(row.filename); + if (!hash && !key) continue; + await sql`UPDATE personal_book SET filenameHash = ${hash}, filenameKey = ${key} WHERE id = ${row.id}`.execute( + db, + ); + } + + const docs = ( + await sql<{ + id: number; + filename: string | null; + }>`SELECT id, filename FROM sync_document WHERE filename IS NOT NULL`.execute(db) + ).rows; + for (const row of docs) { + const key = filenameKey(row.filename); + if (!key) continue; + await sql`UPDATE sync_document SET filenameKey = ${key} WHERE id = ${row.id}`.execute(db); + } + }, + async down(db: Kysely) { + await sql`DROP INDEX IF EXISTS idx_sync_document_user_filename_key`.execute(db); + await sql`DROP INDEX IF EXISTS idx_personal_book_user_filename_key`.execute(db); + await sql`DROP INDEX IF EXISTS idx_personal_book_user_filename_hash`.execute(db); + await db.schema.alterTable("sync_document").dropColumn("filenameKey").execute(); + await db.schema.alterTable("personal_book").dropColumn("filenameKey").execute(); + await db.schema.alterTable("personal_book").dropColumn("filenameHash").execute(); + }, +}; + +migrations["023"] = { + async up(db: Kysely) { + // Covering index for the storage quota. The quota is enforced as + // `SUM(sizeBytes) WHERE userDid = ?` evaluated inside the upload INSERT, and + // the existing `idx_personal_book_user` only covers `userDid` — SQLite would + // walk it and then fetch every row from the table to read `sizeBytes`. With + // the size in the index the SUM is an index-only range scan. + await sql`CREATE INDEX idx_personal_book_user_size ON personal_book(userDid, sizeBytes)`.execute( + db, + ); + + // `parseBook` returns `authors: ""` (not null) on every fallback path, and + // the web upload route stored that verbatim while the XRPC one normalised + // it. The two are indistinguishable to JS truthiness and completely + // different to SQL — `WHERE authors IS NULL` silently misses every row the + // web route wrote. Normalise the existing rows once here; the shared upload + // core writes NULL from now on. + await sql`UPDATE personal_book SET authors = NULL WHERE authors = ''`.execute(db); + await sql`UPDATE personal_book SET language = NULL WHERE language = ''`.execute(db); + }, + async down(db: Kysely) { + await sql`DROP INDEX IF EXISTS idx_personal_book_user_size`.execute(db); + }, +}; + // APIs export const createDb = (location: string): { db: Database; sqlite: DatabaseSync } => { diff --git a/src/env.ts b/src/env.ts index a4351c77..db3521fe 100644 --- a/src/env.ts +++ b/src/env.ts @@ -1,4 +1,4 @@ -import { cleanEnv, num, port, str, testOnly } from "envalid"; +import { bool, cleanEnv, num, port, str, testOnly } from "envalid"; // Bun loads .env automatically; envalid reads process.env @@ -78,4 +78,32 @@ export const env = cleanEnv(process.env, { default: "", desc: "Hex-encoded imgproxy signing salt (IMGPROXY_SALT). Empty uses unsafe URLs (dev only).", }), + XRPC_SERVICE_AUTH: bool({ + default: true, + desc: "Accept atproto inter-service auth (Authorization: Bearer ) on /xrpc/*, which is what lets non-browser clients use the personal library. Kill switch only; the iron-session cookie path is unaffected by it.", + }), + XRPC_SERVICE_AUTH_MAX_AGE: num({ + default: 3600, + desc: "Maximum accepted service-JWT lifetime window, in seconds. atcute defaults to 300, but a PDS mints up to 3600 when `lxm` is set and most client SDKs don't expose `exp`, so those tokens would be refused as JwtTooOld. The token's own `exp` is still enforced separately.", + }), + XRPC_SERVICE_AUTH_REPLAY: bool({ + default: false, + desc: "Reject reused (iss, jti) service tokens. Off by default: enabling it forces a fresh com.atproto.server.getServiceAuth round-trip to the user's PDS on every single call, to close a <=MAX_AGE window on a token already scoped to one lxm and one audience.", + }), + XRPC_SERVICE_AUTH_AUDIENCES: str({ + default: "", + desc: "Comma-separated `aud` values this deployment answers for. Empty uses BOOKHIVE_DID plus its #bookhive_appview fragment. Matching is exact string equality — a bare DID does not match a fragment audience.", + }), + LIBRARY_DIR: str({ + default: "", + desc: "Root directory for personal-library files. Empty derives it from dirname(DB_PATH)/library. Set explicitly to put the library on a different volume from the DB — and by the test preload, so tests can never write ebooks into the repo.", + }), + PERSONAL_LIBRARY_QUOTA_BYTES: num({ + default: 2 * 1024 * 1024 * 1024, + desc: "Total bytes of personal-library files one user may store. Enforced as SUM(personal_book.sizeBytes) evaluated *inside* the INSERT, so two concurrent uploads can't both observe the pre-insert total. The per-file ceiling (MAX_PERSONAL_BOOK_BYTES, 100 MB) applies on top. Excludes stored cover images, which are <1% of the total.", + }), + UPLOAD_PARSE_CONCURRENCY: num({ + default: 2, + desc: "Per-process cap on concurrent ebook metadata parses. The parse is the only step that holds the whole file (<=100 MB) in native memory, so this is the memory bound on uploads — and it is per-process: with WEB_CONCURRENCY=4 the cluster-wide ceiling is this x 4 x 100 MB. See src/utils/uploadPersonalBook.ts.", + }), }); diff --git a/src/pages/library.test.tsx b/src/pages/library.test.tsx index e1595866..1a208ceb 100644 --- a/src/pages/library.test.tsx +++ b/src/pages/library.test.tsx @@ -55,6 +55,45 @@ describe("LibraryPage", () => { }); }); + describe("upload error alert", () => { + it("renders the reason a plain form post failed, in both layouts", async () => { + // A post can't read a JSON error body, so the browser path + // redirects with a code. Before this it landed on a page showing raw + // JSON as text. + for (const bookCount of [0, 3]) { + const html = await render( + , + ); + expect(html).toContain("Your library is full"); + expect(html).toContain('role="alert"'); + } + }); + + it("falls back to a generic message for an unknown code", async () => { + const html = await render( + , + ); + expect(html).toContain("didn't work"); + }); + + it("renders no alert when there is no error", async () => { + const html = await render( + , + ); + expect(html).not.toContain('role="alert"'); + }); + }); + it("uses the populated layout when only synced documents exist", async () => { // Progress can arrive from an e-reader before anything is uploaded; that // still needs the manager so the user can triage those documents. diff --git a/src/pages/library.tsx b/src/pages/library.tsx index f4ec60c2..1d37dc76 100644 --- a/src/pages/library.tsx +++ b/src/pages/library.tsx @@ -296,11 +296,38 @@ const LibraryDialog: FC<{ ); +/** + * What a failed upload said, for the codes `POST /library/upload` redirects + * with. A plain `` post can't read a JSON error body, so the browser path + * round-trips a code and renders it here — it used to land on a page showing + * raw JSON as text. + */ +const UPLOAD_ERRORS: Record = { + TooLarge: "That file is larger than the 100 MB limit.", + QuotaExceeded: "Your library is full. Delete a book to free up space, then try again.", + UnsupportedFormat: "That file isn't a supported ebook. Try EPUB, MOBI, AZW3, FB2 or CBZ.", + AlreadyExists: "That book is already in your library.", + EmptyFile: "That file is empty.", + NoFile: "No file was selected.", + Busy: "The server is busy right now — try that upload again in a moment.", +}; + +const UploadError: FC<{ code: string }> = ({ code }) => ( +

+); + export const LibraryPage: FC<{ handle: string; bookCount: number; syncDocCount: number; -}> = ({ handle, bookCount, syncDocCount }) => { + /** `?error=` code from a failed upload redirect, if any. */ + uploadError?: string | undefined; +}> = ({ handle, bookCount, syncDocCount, uploadError }) => { // Nothing uploaded and nothing synced: there's no library to manage yet, so // explain the feature and put setup right on the page instead of behind // buttons the user has no reason to press. @@ -335,6 +362,11 @@ export const LibraryPage: FC<{

Supported formats: EPUB, MOBI, AZW3, FB2, CBZ.

+ {uploadError ? ( +
+ +
+ ) : null} @@ -366,6 +398,12 @@ export const LibraryPage: FC<{ + {uploadError ? ( +
+ +
+ ) : null} + () ctx.addWideEventContext({ backfill_catalog: "started" }); return c.json({ message: "Backfill started" }, 202); }) - .get("/backfill-catalog/progress", (c) => { + .get("/backfill-catalog/progress", async (c) => { const authorization = c.req.header("authorization"); if ( !env.EXPORT_SHARED_SECRET || @@ -56,7 +56,10 @@ const admin = new Hono() return c.json({ message: "Not Found" }, 404); } - return c.json(getBackfillProgress()); + // Reads through to the KV, so this answers usefully even when the request + // lands on a worker that never ran the backfill, or after a restart. + const ctx = c.get("ctx"); + return c.json(await getBackfillProgress(ctx.kv)); }) .get("/export", async (c) => { const ctx = c.get("ctx"); diff --git a/src/routes/library.test.ts b/src/routes/library.test.ts index ede9c4df..b1530451 100644 --- a/src/routes/library.test.ts +++ b/src/routes/library.test.ts @@ -9,6 +9,7 @@ import { migrateToLatest, type DatabaseSchema, type Database } from "../db"; import type { HiveId } from "../types"; import { koreaderPartialMD5 } from "../utils/bookMetadata/index"; import { NO_HIVE_MATCH } from "../utils/syncMatching"; +import { filenameKey, koreaderFilenameHash } from "../utils/filenameMatching"; import libraryRouter from "./library"; type TestApp = Hono; @@ -56,8 +57,10 @@ async function seedSyncDocument( title?: string | null; percentage?: number; userDid?: string; + filename?: string; }, ) { + const filename = opts.filename ?? "book.epub"; await db .insertInto("sync_document") .values({ @@ -65,7 +68,8 @@ async function seedSyncDocument( provider: "kosync", documentHash: opts.documentHash, hiveId: opts.hiveId ?? null, - filename: "book.epub", + filename, + filenameKey: filenameKey(filename), title: opts.title ?? "A Synced Book", authors: "An Author", progressData: JSON.stringify({ @@ -81,14 +85,21 @@ async function seedSyncDocument( .execute(); } -async function seedPersonalBook(db: Database, contentHash: string, filePath: string) { +async function seedPersonalBook( + db: Database, + contentHash: string, + filePath: string, + filename = "book.epub", +) { await db .insertInto("personal_book") .values({ userDid: DID, contentHash, hiveId: null, - filename: "book.epub", + filename, + filenameHash: koreaderFilenameHash(filename), + filenameKey: filenameKey(filename), title: "An Uploaded Book", authors: "An Author", language: "en", @@ -149,6 +160,55 @@ describe("GET /library/sync/documents", () => { expect(doc?.hasFile).toBe(true); }); + it("reports hasFile=true for a client using the FILENAME checksum method", async () => { + // KOSync's other checksum mode sends md5(basename) as the document id, so + // it never equals our content hash. Matching only on content hash left + // every one of these users' uploads looking unsynced. + const filename = "The Dispossessed.epub"; + await seedSyncDocument(db, { documentHash: koreaderFilenameHash(filename)!, filename }); + await seedPersonalBook(db, "some-content-hash", "/tmp/nonexistent.epub", filename); + + const [doc] = await getDocuments(app); + expect(doc?.hasFile).toBe(true); + }); + + it("reports hasFile=true when conversion changed the bytes and the extension", async () => { + // Neither hash can line up: the device holds an .azw3 calibre made from the + // .epub we hold. The readable filename is the only thing left. + await seedSyncDocument(db, { + documentHash: "hash-from-the-converted-copy", + filename: "Dune - Frank Herbert.azw3", + }); + await seedPersonalBook(db, "hash-of-original", "/tmp/x.epub", "dune_-_frank_herbert.epub"); + + const [doc] = await getDocuments(app); + expect(doc?.hasFile).toBe(true); + }); + + it("lists a document once even when several uploads match it", async () => { + // EXISTS rather than a join: a join emits one row per match. + const filename = "Dune.epub"; + await seedSyncDocument(db, { documentHash: koreaderFilenameHash(filename)!, filename }); + await seedPersonalBook(db, "hash-a", "/tmp/a.epub", filename); + await seedPersonalBook(db, "hash-b", "/tmp/b.epub", "Dune.azw3"); + + const docs = await getDocuments(app); + expect(docs).toHaveLength(1); + expect(docs[0]?.hasFile).toBe(true); + }); + + it("does not match a file with no filename key to a document with none", async () => { + // Both keys null; `NULL = NULL` is not true, which is what stops every + // metadata-less document matching every metadata-less upload. + await seedSyncDocument(db, { documentHash: "hash-orphan" }); + await db.updateTable("sync_document").set({ filenameKey: null }).execute(); + await seedPersonalBook(db, "other-hash", "/tmp/nonexistent.epub"); + await db.updateTable("personal_book").set({ filenameKey: null, filenameHash: null }).execute(); + + const [doc] = await getDocuments(app); + expect(doc?.hasFile).toBe(false); + }); + it("does not match another user's uploaded file", async () => { await seedSyncDocument(db, { documentHash: "hash-shared" }); await seedPersonalBook(db, "hash-shared", "/tmp/nonexistent.epub"); @@ -456,7 +516,9 @@ describe("POST /library/upload", () => { db = await createTestDb(); app = createApp(db); // Seed the row the uploader will collide with, keyed by the same hash the - // upload computes, so neither request reaches the filesystem. + // upload computes, so neither request reaches the library directory. This + // relies on the shared core running its duplicate check before the parse + // and before the rename — keep that ordering. await seedPersonalBook(db, koreaderPartialMD5(new TextEncoder().encode(FB2)), "/tmp/dupe.fb2"); }); @@ -466,10 +528,12 @@ describe("POST /library/upload", () => { expect((await res.json()) as { error: string }).toHaveProperty("error"); }); - it("still redirects the browser back to the library", async () => { + it("redirects the browser back to the library with the reason", async () => { + // A plain post used to get raw JSON rendered as a text page. Every + // failure now round-trips a code the library page can render as an alert. const res = await uploadRequest(app, { json: false }); expect(res.status).toBe(302); - expect(res.headers.get("location")).toBe("/library"); + expect(res.headers.get("location")).toBe("/library?error=AlreadyExists"); }); it("401s without a session", async () => { diff --git a/src/routes/library.tsx b/src/routes/library.tsx index 2e8de021..4c14d33e 100644 --- a/src/routes/library.tsx +++ b/src/routes/library.tsx @@ -1,6 +1,6 @@ import { zValidator } from "@hono/zod-validator"; import { Hono } from "hono"; -import { bodyLimit } from "hono/body-limit"; +import type { ContentfulStatusCode } from "hono/utils/http-status"; import { z } from "zod"; import type { AppEnv } from "../context"; @@ -9,25 +9,27 @@ import { currentSyncPassword, rotateSyncToken } from "../middleware/sync-auth"; import { bridgeProgressToUserBook } from "../utils/syncBridge"; import { updateBookRecord } from "../utils/getBook"; import { READING } from "../constants"; -import { - detectFormat, - parseBook, - koreaderPartialMD5, - isUsableCover, -} from "../utils/bookMetadata/index"; -import { - ensureDir, - personalBookDir, - bookFilePath, - coverFilePath, - streamPersonalBook, - MAX_PERSONAL_BOOK_BYTES, -} from "../utils/personalLibrary"; -import { NO_HIVE_MATCH } from "../utils/syncMatching"; +import { streamPersonalBook, MAX_PERSONAL_BOOK_BYTES } from "../utils/personalLibrary"; +import { uploadPersonalBook } from "../utils/uploadPersonalBook"; +import { NO_HIVE_MATCH, SAME_BOOK_FILE } from "../utils/syncMatching"; import type { HiveId, SyncProgressData } from "../types"; const MAX_FILE_SIZE = MAX_PERSONAL_BOOK_BYTES; +/** + * Headroom for multipart part headers and boundaries when checking the request's + * total `Content-Length` against the per-file limit. Generous on purpose — this + * is a cheap early reject, and the core enforces the real cap on the file itself. + */ +const MULTIPART_SLACK = 64 * 1024; + +/** Human-readable byte count for user-facing quota messages. */ +function formatBytes(bytes: number): string { + if (bytes >= 1024 ** 3) return `${(bytes / 1024 ** 3).toFixed(1)} GB`; + if (bytes >= 1024 ** 2) return `${Math.round(bytes / 1024 ** 2)} MB`; + return `${Math.round(bytes / 1024)} KB`; +} + const app = new Hono() .get("/", async (c) => { const agent = await c.get("ctx").getSessionAgent(); @@ -57,156 +59,83 @@ const app = new Hono() handle={handle} bookCount={Number(books.total)} syncDocCount={Number(documents.total)} + uploadError={c.req.query("error")} />, { title: "Personal Library" }, ); }) - .post( - "/upload", - // Rejects on Content-Length, and aborts the stream once the cap is passed - // when there is none. This has to run *before* the handler because - // `c.req.formData()` materialises the entire multipart body in native - // memory — the `file.size` check below cannot fire until after that has - // already happened, so on its own it bounds what we store, not what we - // allocate. - bodyLimit({ - maxSize: MAX_FILE_SIZE, - onError: (c) => c.json({ error: "File exceeds 100 MB limit" }, 413), - }), - async (c) => { - const agent = await c.get("ctx").getSessionAgent(); - if (!agent) return c.json({ error: "Unauthorized" }, 401); - - // The browser posts a plain and wants to land back on the library; - // the mobile app posts the same multipart body but needs the created record - // (and a real status on duplicates) rather than a redirect to HTML. - const wantsJson = c.req.header("accept")?.includes("application/json") ?? false; - - const formData = await c.req.formData(); - const file = formData.get("file"); - if (!file || !(file instanceof File)) { - return c.json({ error: "No file provided" }, 400); - } - - // Check the declared size *before* materialising the file. The check used - // to run after `arrayBuffer()`, so rejecting an oversized upload still cost - // a full copy of it in native memory first. - if (file.size > MAX_FILE_SIZE) { - return c.json({ error: "File exceeds 100 MB limit" }, 413); - } - - const bytes = new Uint8Array(await file.arrayBuffer()); - if (bytes.length > MAX_FILE_SIZE) { - return c.json({ error: "File exceeds 100 MB limit" }, 413); - } - - const formatInfo = detectFormat(bytes, file.name); - if (formatInfo.format === "unknown") { - return c.json({ error: "Unsupported file format" }, 400); - } - - const contentHash = koreaderPartialMD5(bytes); - - const { db } = c.get("ctx"); - - // Check for duplicate - const existing = await db - .selectFrom("personal_book") - .select("id") - .where("userDid", "=", agent.did) - .where("contentHash", "=", contentHash) - .executeTakeFirst(); - if (existing) { - if (wantsJson) { - return c.json({ error: "This book is already in your library" }, 409); - } - return c.redirect("/library"); - } - - const metadata = parseBook(bytes, file.name); - - await ensureDir(personalBookDir(agent.did, contentHash)); - - const filePath = bookFilePath(agent.did, contentHash, formatInfo.ext); - await Bun.write(filePath, bytes); - - let coverPath: string | null = null; - let coverMime: string | null = null; - if (metadata.cover && (await isUsableCover(metadata.cover.bytes))) { - const cp = coverFilePath(agent.did, contentHash, metadata.cover.ext); - await Bun.write(cp, metadata.cover.bytes); - coverPath = cp; - coverMime = metadata.cover.mime; - } - - // Try to match an existing sync_document (contentHash is the KOReader partial MD5) - let matchedHiveId: HiveId | null = null; - const syncDoc = await db - .selectFrom("sync_document") - .select(["hiveId"]) - .where("userDid", "=", agent.did) - .where("documentHash", "=", contentHash) - .executeTakeFirst(); - if (syncDoc?.hiveId) { - matchedHiveId = syncDoc.hiveId; - } - - const now = new Date().toISOString(); - await db - .insertInto("personal_book") - .values({ - userDid: agent.did, - contentHash, - hiveId: matchedHiveId, - filename: file.name, - title: metadata.title, - authors: metadata.authors, - language: metadata.language || null, - format: formatInfo.format, - mime: formatInfo.mime, - filePath, - coverPath, - coverMime, - sizeBytes: bytes.length, - createdAt: now, - updatedAt: now, - }) - .execute(); - - // Mark the book as owned if auto-linked and user has it in their library - if (matchedHiveId) { - await db - .updateTable("user_book") - .set({ owned: 1 }) - .where("userDid", "=", agent.did) - .where("hiveId", "=", matchedHiveId) - .where("owned", "=", 0) - .execute(); - } - - if (wantsJson) { - // Same shape as getPersonalLibrary#personalBookView so clients have one - // book type for both the list and the upload response. - return c.json({ - book: { - contentHash, - title: metadata.title, - authors: metadata.authors || undefined, - language: metadata.language || undefined, - format: formatInfo.format, - mime: formatInfo.mime, - sizeBytes: bytes.length, - createdAt: now, - updatedAt: now, - hiveId: matchedHiveId ?? undefined, - coverUrl: coverPath ? `/library/covers/${contentHash}` : undefined, - }, - }); - } - - return c.redirect("/library"); - }, - ); + // Thin adapter over `uploadPersonalBook` — the same core the XRPC procedure + // calls. Everything here is transport: content negotiation and the mapping + // from the core's discriminated result to a status code. + // + // Note there is no `bodyLimit()` middleware any more. It only short-circuits + // on `Content-Length`; given a chunked body it drains the whole stream into + // an array and rebuilds the Request, so a compliant 100 MB chunked upload was + // buffered there *and again* by `formData()`. The core caps while streaming + // to disk, which bounds every path at one chunk. + .post("/upload", async (c) => { + const agent = await c.get("ctx").getSessionAgent(); + if (!agent) return c.json({ error: "Unauthorized" }, 401); + + // The browser posts a plain and wants to land back on the library; + // the mobile app posts the same multipart body but needs the created record + // (and a real status on duplicates) rather than a redirect to HTML. + const wantsJson = c.req.header("accept")?.includes("application/json") ?? false; + const fail = (status: ContentfulStatusCode, code: string, error: string, extra = {}) => + wantsJson + ? c.json({ error, code, ...extra }, status) + : c.redirect(`/library?error=${encodeURIComponent(code)}`); + + // The early reject `bodyLimit` used to give. `formData()` below still + // materialises the File in native memory — Bun/hono expose no incremental + // multipart API — so refusing an obviously oversized body before parsing it + // is worth the two lines. MULTIPART_SLACK covers the part headers. + const declaredTotal = Number(c.req.header("content-length")); + if (Number.isFinite(declaredTotal) && declaredTotal > MAX_FILE_SIZE + MULTIPART_SLACK) { + return fail(413, "TooLarge", "File exceeds 100 MB limit"); + } + + const formData = await c.req.formData(); + const file = formData.get("file"); + if (!file || !(file instanceof File)) { + return fail(400, "NoFile", "No file provided"); + } + + const { db, kv } = c.get("ctx"); + const result = await uploadPersonalBook({ + db, + kv, + userDid: agent.did, + filename: file.name, + source: { kind: "stream", body: file.stream(), declaredLength: file.size }, + }); + + if (result.ok) { + // Same shape as getPersonalLibrary#personalBookView so clients have one + // book type for both the list and the upload response. + return wantsJson ? c.json({ book: result.book }) : c.redirect("/library"); + } + + switch (result.reason) { + case "too-large": + return fail(413, "TooLarge", "File exceeds 100 MB limit"); + case "quota-exceeded": + return fail( + 413, + "QuotaExceeded", + `Library full — ${formatBytes(result.usedBytes)} of ${formatBytes(result.quotaBytes)} used. Delete a book to free space.`, + { usedBytes: result.usedBytes, quotaBytes: result.quotaBytes }, + ); + case "unsupported-format": + return fail(400, "UnsupportedFormat", "Unsupported file format"); + case "duplicate": + return fail(409, "AlreadyExists", "This book is already in your library"); + case "empty": + return fail(400, "EmptyFile", "The file is empty"); + case "busy": + return fail(503, "Busy", "Server is busy — try again in a moment"); + } + }); // Serve cover images for personal library books app.get("/covers/:hash", async (c) => { @@ -316,13 +245,6 @@ app.get("/sync/documents", async (c) => { const rows = await db .selectFrom("sync_document") .leftJoin("hive_book", "hive_book.id", "sync_document.hiveId") - // A document whose hash matches an uploaded file is the same book: the - // library grid renders it, so the sync sections must not claim it too. - .leftJoin("personal_book", (join) => - join - .onRef("personal_book.contentHash", "=", "sync_document.documentHash") - .onRef("personal_book.userDid", "=", "sync_document.userDid"), - ) .select([ "sync_document.documentHash as document", "sync_document.title as title", @@ -332,8 +254,22 @@ app.get("/sync/documents", async (c) => { "sync_document.updatedAt as updatedAt", "sync_document.hiveId as hiveId", "hive_book.title as bookTitle", - "personal_book.id as personalBookId", ]) + // A document we hold the file for is the same book: the library grid + // renders it, so the sync sections must not claim it too. An EXISTS rather + // than a join because more than one upload can match one document (see + // SAME_BOOK_FILE) and a join would list the document once per match. + .select((eb) => + eb + .exists( + eb + .selectFrom("personal_book") + .select("personal_book.id") + .whereRef("personal_book.userDid", "=", "sync_document.userDid") + .where(SAME_BOOK_FILE), + ) + .as("hasFile"), + ) .where("sync_document.userDid", "=", agent.did) .orderBy("sync_document.updatedAt", "desc") .execute(); @@ -362,7 +298,7 @@ app.get("/sync/documents", async (c) => { hiveId: dismissed ? null : row.hiveId, bookTitle: dismissed ? null : row.bookTitle, dismissed, - hasFile: row.personalBookId != null, + hasFile: Boolean(row.hasFile), }; }); diff --git a/src/routes/opds.test.ts b/src/routes/opds.test.ts index 35737c98..6a147e07 100644 --- a/src/routes/opds.test.ts +++ b/src/routes/opds.test.ts @@ -309,4 +309,41 @@ describe("OPDS 2.0 content negotiation", () => { expect(res.status).toBe(401); expect(res.headers.get("www-authenticate")).toContain("Basic"); }); + + describe("cover caching", () => { + // This route sits under `/opds/books/`, which is excluded from hono's + // etag() middleware, so if it doesn't set a validator itself it cannot ever + // answer a conditional request. It didn't: production served 43 cover + // fetches in 48h and never once returned a 304, while a catalogue browse + // re-requests every cover on the page. + const coverPath = "/tmp/bookhive-opds-cover-test.jpg"; + + beforeEach(async () => { + await Bun.write(coverPath, "not-really-a-jpeg-but-bytes-are-bytes"); + await seedBook(db, { contentHash: "hash-a", coverPath }); + }); + + it("serves the cover with a strong ETag", async () => { + const res = await app.request("/opds/books/hash-a/cover", { + headers: { authorization: auth }, + }); + expect(res.status).toBe(200); + expect(res.headers.get("etag")).toBe('"hash-a-cover"'); + }); + + it("answers a matching If-None-Match with 304 and no body", async () => { + const res = await app.request("/opds/books/hash-a/cover", { + headers: { authorization: auth, "if-none-match": '"hash-a-cover"' }, + }); + expect(res.status).toBe(304); + expect(await res.text()).toBe(""); + }); + + it("still serves the bytes when the validator does not match", async () => { + const res = await app.request("/opds/books/hash-a/cover", { + headers: { authorization: auth, "if-none-match": '"something-else"' }, + }); + expect(res.status).toBe(200); + }); + }); }); diff --git a/src/routes/opds.ts b/src/routes/opds.ts index bdfe4cfe..2cab338e 100644 --- a/src/routes/opds.ts +++ b/src/routes/opds.ts @@ -2,7 +2,7 @@ import { Hono, type Context } from "hono"; import type { AppEnv } from "../context"; import { opdsAuthMiddleware } from "../middleware/opds-auth"; import { escapeXml } from "../utils/xml"; -import { OPDS_PAGE_SIZE, streamPersonalBook } from "../utils/personalLibrary"; +import { etagMatches, OPDS_PAGE_SIZE, streamPersonalBook } from "../utils/personalLibrary"; import type { Selectable } from "kysely"; import type { PersonalBookRow } from "../types"; @@ -19,7 +19,20 @@ const THUMBNAIL_REL = "http://opds-spec.org/image/thumbnail"; type OpdsEnv = AppEnv & { Variables: { opdsUserDid: string } }; -type BookForEntry = Selectable & { +// Only the columns an entry actually renders, so adding one to `personal_book` +// (filename matching, say) does not oblige every feed query to select it. +type BookForEntry = Pick< + Selectable, + | "contentHash" + | "hiveId" + | "title" + | "authors" + | "language" + | "mime" + | "coverPath" + | "coverMime" + | "updatedAt" +> & { hiveBookCover?: string | null; hiveBookDescription?: string | null; }; @@ -555,10 +568,22 @@ app.get("/books/:hash/cover", async (c) => { if (book.coverPath) { const file = Bun.file(book.coverPath); if (await file.exists()) { - return c.body(file.stream(), 200, { + // This route is under `/opds/books/`, which is in ETAG_EXCLUDED_PREFIXES, + // so hono's etag() never sees it — without an ETag set here the response + // carries no validator at all and a 304 is impossible. Production bore + // that out: 43 cover fetches over 48h, none of them conditional, while + // the catalogue root was hit 431 times. The hash is immutable content, so + // the validator is free. + const etag = `"${hash}-cover"`; + const headers = { "Content-Type": book.coverMime || "image/jpeg", "Cache-Control": "private, max-age=86400", - }); + ETag: etag, + }; + if (etagMatches(c.req.header("if-none-match"), etag)) { + return c.body(null, 304, headers); + } + return c.body(file.stream(), 200, headers); } } diff --git a/src/routes/sync/kosync.test.ts b/src/routes/sync/kosync.test.ts new file mode 100644 index 00000000..a0411de0 --- /dev/null +++ b/src/routes/sync/kosync.test.ts @@ -0,0 +1,283 @@ +/** + * End-to-end cover for the bug PR #204 reported: an ebook uploaded *before* the + * first KOSync push left `sync_document.hiveId` null forever, so reading + * progress never reached the user's public book. + * + * The upload's writeback couldn't help — the document didn't exist yet — and the + * KOSync handler only auto-matched on title/author, which a default-configured + * KOReader never sends. `matchSyncDocumentForUser` closes it from the other + * side: the document hash *is* the uploaded file's content hash, so the file's + * own metadata (and any book already linked to it) resolves the document. + * + * These tests drive the real route, so they fail if either half regresses. + */ + +import { describe, it, expect, beforeEach, afterEach } from "bun:test"; +import { Database as DatabaseSync } from "bun:sqlite"; +import { Hono } from "hono"; +import { Kysely, SqliteDialect } from "kysely"; +import { createStorage } from "unstorage"; +import memoryDriver from "unstorage/drivers/memory"; +import { rm } from "node:fs/promises"; +import path from "node:path"; +import type { Storage } from "unstorage"; + +import { wrapBunSqliteForKysely } from "../../bun-sqlite-kysely"; +import type { AppContext, AppEnv } from "../../context"; +import { migrateToLatest, type DatabaseSchema, type Database } from "../../db"; +import type { HiveId } from "../../types"; +import { currentSyncPassword } from "../../middleware/sync-auth"; +import { getHiveId } from "../../scrapers/getHiveId"; +import { koreaderPartialMD5 } from "../../utils/bookMetadata/index"; +import { makeEpub } from "../../utils/bookMetadata/testFixtures"; +import { personalBookDir } from "../../utils/personalLibrary"; +import { uploadPersonalBook } from "../../utils/uploadPersonalBook"; +import { NO_HIVE_MATCH } from "../../utils/syncMatching"; +import kosyncRouter from "./kosync"; + +const DID = "did:plc:testuser"; +const HANDLE = "alice.bsky.social"; + +let db: Database; +let kv: Storage; + +async function createTestDb(): Promise { + const sqlite = new DatabaseSync(":memory:"); + sqlite.exec("PRAGMA journal_mode = WAL"); + const database = new Kysely({ + dialect: new SqliteDialect({ database: wrapBunSqliteForKysely(sqlite) }), + }); + await migrateToLatest(database, sqlite); + return database; +} + +function createApp(): Hono { + const app = new Hono(); + app.use("*", async (c, next) => { + c.set("ctx", { + db, + kv, + baseIdResolver: { + handle: { resolve: async (h: string) => (h === HANDLE ? DID : null) }, + }, + addWideEventContext: () => {}, + } as unknown as AppContext); + await next(); + }); + app.route("/kosync", kosyncRouter); + return app; +} + +/** KOSync sends md5 of the derived password, not the password itself. */ +async function authHeaders(): Promise> { + const password = await currentSyncPassword(kv, DID); + return { + "x-auth-user": HANDLE, + "x-auth-key": new Bun.CryptoHasher("md5").update(password).digest("hex"), + "content-type": "application/json", + }; +} + +/** Push progress the way a default-configured KOReader does: hash and nothing else. */ +async function pushProgress( + app: Hono, + document: string, + percentage: number, + metadata?: { filename?: string; title?: string; authors?: string }, +) { + return app.request("/kosync/syncs/progress", { + method: "PUT", + headers: await authHeaders(), + body: JSON.stringify({ + document, + progress: "/body/DocFragment[3]", + percentage, + device: "kindle", + device_id: "dev-1", + ...(metadata ? { metadata } : {}), + }), + }); +} + +async function seedHiveBook(title: string, authors: string): Promise { + const id = getHiveId({ title, authors }); + await db + .insertInto("hive_book") + .values({ + id, + title, + rawTitle: title, + authors, + source: "goodreads", + thumbnail: "", + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + } as never) + .execute(); + return id; +} + +async function seedUserBook(hiveId: HiveId): Promise { + await db + .insertInto("user_book") + .values({ + uri: `at://${DID}/buzz.bookhive.book/${hiveId}`, + cid: "cid", + userDid: DID, + hiveId, + title: "t", + authors: "a", + status: "buzz.bookhive.defs#reading", + owned: 0, + createdAt: "2026-08-01T00:00:00.000Z", + indexedAt: "2026-08-01T00:00:00.000Z", + }) + .execute(); +} + +beforeEach(async () => { + db = await createTestDb(); + kv = createStorage({ driver: memoryDriver() }); +}); + +afterEach(async () => { + await rm(path.dirname(personalBookDir(DID, "x")), { recursive: true, force: true }).catch( + () => {}, + ); +}); + +describe("PUT /kosync/syncs/progress — upload first, then sync", () => { + it("bridges progress to the public book for a client sending only a hash", async () => { + // This is the reported bug, in order: the file is uploaded, the catalogue + // book exists, and only then does the device push — with no metadata at all. + const hiveId = await seedHiveBook("Dune", "Frank Herbert"); + await seedUserBook(hiveId); + + const bytes = makeEpub({ title: "Dune", authors: ["Frank Herbert"] }); + const upload = await uploadPersonalBook({ + db, + kv, + userDid: DID, + filename: "Dune.epub", + source: { + kind: "stream", + body: new Blob([bytes as BlobPart]).stream(), + declaredLength: bytes.length, + }, + }); + expect(upload.ok).toBe(true); + + const res = await pushProgress(createApp(), koreaderPartialMD5(bytes), 0.42); + expect(res.status).toBe(200); + + const doc = await db.selectFrom("sync_document").select("hiveId").executeTakeFirstOrThrow(); + expect(doc.hiveId).toBe(hiveId); + + const userBook = await db + .selectFrom("user_book") + .select(["bookProgress", "owned"]) + .executeTakeFirstOrThrow(); + expect(JSON.parse(userBook.bookProgress!).percent).toBe(42); + expect(userBook.owned).toBe(1); + }); + + it("still bridges when the upload had already resolved the book itself", async () => { + // The file carries the link; the document inherits it rather than + // re-deriving it from metadata the client never sent. + const hiveId = await seedHiveBook("Neuromancer", "William Gibson"); + await seedUserBook(hiveId); + + const bytes = makeEpub({ title: "Neuromancer", authors: ["William Gibson"] }); + await uploadPersonalBook({ + db, + kv, + userDid: DID, + filename: "Neuromancer.epub", + source: { + kind: "stream", + body: new Blob([bytes as BlobPart]).stream(), + declaredLength: bytes.length, + }, + }); + const file = await db.selectFrom("personal_book").select("hiveId").executeTakeFirstOrThrow(); + expect(file.hiveId).toBe(hiveId); + + await pushProgress(createApp(), koreaderPartialMD5(bytes), 0.9); + + const userBook = await db + .selectFrom("user_book") + .select("bookProgress") + .executeTakeFirstOrThrow(); + expect(JSON.parse(userBook.bookProgress!).percent).toBe(90); + }); + + it("does not overwrite a link the user set by hand", async () => { + const theirs = await seedHiveBook("Dune", "Frank Herbert"); + const manual = "bk_manualchoice" as HiveId; + const bytes = makeEpub({ title: "Dune", authors: ["Frank Herbert"] }); + + await db + .insertInto("sync_document") + .values({ + userDid: DID, + provider: "kosync", + documentHash: koreaderPartialMD5(bytes), + hiveId: manual, + progressData: "{}", + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + }) + .execute(); + + await pushProgress(createApp(), koreaderPartialMD5(bytes), 0.5); + + const doc = await db.selectFrom("sync_document").select("hiveId").executeTakeFirstOrThrow(); + expect(doc.hiveId).toBe(manual); + expect(doc.hiveId).not.toBe(theirs); + }); + + it("respects a dismissed document and never bridges onto the sentinel", async () => { + const hiveId = await seedHiveBook("Dune", "Frank Herbert"); + await seedUserBook(hiveId); + const bytes = makeEpub({ title: "Dune", authors: ["Frank Herbert"] }); + + await db + .insertInto("sync_document") + .values({ + userDid: DID, + provider: "kosync", + documentHash: koreaderPartialMD5(bytes), + hiveId: NO_HIVE_MATCH, + progressData: "{}", + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + }) + .execute(); + + const res = await pushProgress(createApp(), koreaderPartialMD5(bytes), 0.5); + expect(res.status).toBe(200); + + const doc = await db.selectFrom("sync_document").select("hiveId").executeTakeFirstOrThrow(); + expect(doc.hiveId).toBe(NO_HIVE_MATCH); + const userBook = await db + .selectFrom("user_book") + .select("bookProgress") + .executeTakeFirstOrThrow(); + expect(userBook.bookProgress).toBeNull(); + }); + + it("401s without valid sync credentials", async () => { + const res = await createApp().request("/kosync/syncs/progress", { + method: "PUT", + headers: { "x-auth-user": HANDLE, "x-auth-key": "wrong", "content-type": "application/json" }, + body: JSON.stringify({ + document: "d", + progress: "p", + percentage: 0.1, + device: "k", + device_id: "1", + }), + }); + expect(res.status).toBe(401); + }); +}); diff --git a/src/routes/sync/kosync.ts b/src/routes/sync/kosync.ts index 1075b100..73b8c873 100644 --- a/src/routes/sync/kosync.ts +++ b/src/routes/sync/kosync.ts @@ -1,7 +1,8 @@ import { Hono } from "hono"; import type { AppEnv } from "../../context"; import { syncAuthMiddleware } from "../../middleware/sync-auth"; -import { matchSyncDocument } from "../../utils/syncMatching"; +import { matchSyncDocumentForUser } from "../../utils/syncMatching"; +import { filenameKey } from "../../utils/filenameMatching"; import { bridgeProgressToUserBook } from "../../utils/syncBridge"; import type { HiveId, SyncProgressData } from "../../types"; @@ -75,7 +76,7 @@ app.put("/syncs/progress", syncAuthMiddleware, async (c) => { .set({ progressData: JSON.stringify(progressData), updatedAt: now, - ...(filename != null ? { filename } : {}), + ...(filename != null ? { filename, filenameKey: filenameKey(filename) } : {}), ...(title != null ? { title } : {}), ...(authors != null ? { authors } : {}), }) @@ -90,6 +91,7 @@ app.put("/syncs/progress", syncAuthMiddleware, async (c) => { documentHash: document, hiveId: null, filename, + filenameKey: filenameKey(filename), title, authors, progressData: JSON.stringify(progressData), @@ -101,8 +103,17 @@ app.put("/syncs/progress", syncAuthMiddleware, async (c) => { let hiveId = existing?.hiveId ?? null; - if (!hiveId && (title || authors)) { - hiveId = await matchSyncDocument(db, { title, authors, filename }); + // Unconditional: a default-configured KOReader sends no metadata at all, and + // `matchSyncDocumentForUser` resolves those from the uploaded file the + // document hash points at. Gating this on the client having sent something is + // what kept both that case and the filename-only case unmatchable. + if (!hiveId) { + hiveId = await matchSyncDocumentForUser(db, userDid, { + documentHash: document, + title, + authors, + filename, + }); if (hiveId) { await db .updateTable("sync_document") @@ -110,6 +121,12 @@ app.put("/syncs/progress", syncAuthMiddleware, async (c) => { .where("userDid", "=", userDid) .where("provider", "=", "kosync") .where("documentHash", "=", document) + // Only fill a genuinely empty link. `hiveId` was read at the top of the + // handler and the match takes a few queries, so a concurrent request — + // or the user linking by hand in another tab — can land in between; + // production runs three worker processes, so this is not theoretical. + // It also can't clobber the NO_HIVE_MATCH dismissal sentinel. + .where("hiveId", "is", null) .execute(); } } diff --git a/src/test/env-setup.ts b/src/test/env-setup.ts new file mode 100644 index 00000000..4725d47b --- /dev/null +++ b/src/test/env-setup.ts @@ -0,0 +1,22 @@ +/** + * Bun test preload — runs before any test file's imports, and therefore before + * `src/env.ts` is evaluated. + * + * This is load-bearing rather than cosmetic. `env` is frozen by envalid at + * import time, so a test can neither assign to it nor usefully mutate + * `process.env` afterwards (ESM imports hoist). Without this file `DB_PATH` + * falls back to its `devDefault` of `":memory:"`, which makes + * `getLibraryDir()` resolve to `path.dirname(":memory:") + "/library"` — + * i.e. `./library` **inside the repo working tree**. The first test that + * actually exercises an upload would write ebooks into the checkout. + * + * Both paths are set with `??=` so an explicit env var (CI, a targeted debug + * run) still wins. + */ +import { tmpdir } from "node:os"; +import path from "node:path"; + +const root = path.join(tmpdir(), "bookhive-test", String(process.pid)); + +process.env["DB_PATH"] ??= path.join(root, "db.sqlite"); +process.env["LIBRARY_DIR"] ??= path.join(root, "library"); diff --git a/src/types.ts b/src/types.ts index e3c76494..6a21588c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -283,6 +283,10 @@ export type PersonalBookRow = { contentHash: string; hiveId: HiveId | null; filename: string; + /** md5 of the basename — the id a FILENAME-checksum KOSync client sends. */ + filenameHash: string | null; + /** Normalized, extension-less basename; survives format conversion. */ + filenameKey: string | null; title: string; authors: string | null; language: string | null; @@ -319,6 +323,8 @@ export type SyncDocumentRow = { documentHash: string; hiveId: HiveId | null; filename: string | null; + /** Normalized `filename`; see src/utils/filenameMatching.ts. */ + filenameKey: string | null; title: string | null; authors: string | null; progressData: string; diff --git a/src/utils/account.ts b/src/utils/account.ts new file mode 100644 index 00000000..c1c1918b --- /dev/null +++ b/src/utils/account.ts @@ -0,0 +1,69 @@ +/** + * "Has this DID ever used BookHive?" + * + * This exists for service auth and nothing else. A valid inter-service JWT + * proves the caller controls an atproto identity — it says nothing about + * whether that identity has any relationship with us. Without a gate here, any + * DID on the network could authenticate and start filling a 2 GB library on our + * disk. + * + * The other credentialled surfaces get this implicitly: OPDS and KOSync derive + * their password from `COOKIE_SECRET`, so the only way to learn it is to have + * signed in and read it off the settings page. Service auth has no such + * property, so the check has to be explicit. + * + * There is no accounts table (see `DatabaseSchema`), so this is a KV marker + * written at sign-in, plus a one-time probe of the durable traces an existing + * account leaves — every user predates the marker. + */ + +import type { Storage } from "unstorage"; +import type { Database } from "../db"; + +const ACCOUNT_PREFIX = "account:"; + +/** Record that a DID has signed in. Called from the OAuth callback. */ +export async function markAccount(kv: Storage, did: string): Promise { + await kv.setItem(`${ACCOUNT_PREFIX}${did}`, 1); +} + +/** + * Marker first; on a miss, backfill from any durable trace of the account. The + * result is memoised permanently — this is not a fact that can become false — + * so the four-way probe runs at most once per DID. + */ +export async function isKnownAccount( + deps: { db: Database; kv: Storage }, + did: string, +): Promise { + const key = `${ACCOUNT_PREFIX}${did}`; + if (await deps.kv.hasItem(key)) return true; + + // All four are indexed on userDid (or are a direct KV key lookup). + const [session, userBook, personalBook, syncDoc] = await Promise.all([ + deps.kv.hasItem(`auth_session:${did}`), + deps.db + .selectFrom("user_book") + .select("uri") + .where("userDid", "=", did) + .limit(1) + .executeTakeFirst(), + deps.db + .selectFrom("personal_book") + .select("id") + .where("userDid", "=", did) + .limit(1) + .executeTakeFirst(), + deps.db + .selectFrom("sync_document") + .select("id") + .where("userDid", "=", did) + .limit(1) + .executeTakeFirst(), + ]); + + if (!session && !userBook && !personalBook && !syncDoc) return false; + + await markAccount(deps.kv, did); + return true; +} diff --git a/src/utils/bookMatching.ts b/src/utils/bookMatching.ts new file mode 100644 index 00000000..3af41ac8 --- /dev/null +++ b/src/utils/bookMatching.ts @@ -0,0 +1,94 @@ +/** + * Fuzzy title/author scoring primitives, shared by anything that has to decide + * whether two differently-written strings name the same book. + * + * Ported from the MIT-licensed shelfcheck project + * (`nowells/libby-reading-list` — `app/lib/libby.ts` and `app/lib/dedupe.ts`) + * with light adaptation for BookHive (tab-separated authors, HiveId). + * + * These normalize on the ASCII range: `normalizeForMatch` drops any character + * outside `[a-z0-9\s]` rather than folding it, so callers with non-Latin input + * must fold diacritics themselves first (`src/utils/filenameMatching.ts` does) + * and must not treat an empty word list as agreement. + */ + +const STOP_WORDS = new Set([ + "a", + "an", + "the", + "and", + "or", + "of", + "in", + "on", + "at", + "to", + "for", + "is", + "it", + "by", + "as", + "be", + "no", + "not", + "but", + "from", + "with", +]); + +/** + * Lowercase, strip non-alphanumerics, collapse whitespace. + * + * Note that punctuation is *deleted*, not replaced with a space, which is the + * point: "hitchhiker's" and "hitchhikers" collapse onto the same word instead + * of differing by a stray "s" token. + */ +export function normalizeForMatch(input: string): string { + return input + .toLowerCase() + .replace(/[^a-z0-9\s]/g, "") + .replace(/\s+/g, " ") + .trim(); +} + +/** Significant words in `s` after stop-word removal. */ +export function contentWords(s: string): string[] { + return normalizeForMatch(s) + .split(" ") + .filter((w) => w.length > 0 && !STOP_WORDS.has(w)); +} + +/** + * Sørensen–Dice on whitespace-separated words after normalization. + * Returns 1 for an exact normalized match. + */ +export function similarityScore(a: string, b: string): number { + const na = normalizeForMatch(a); + const nb = normalizeForMatch(b); + if (na === nb) return 1; + + const wordsA = na.split(" ").filter(Boolean); + const wordsB = nb.split(" ").filter(Boolean); + if (wordsA.length === 0 || wordsB.length === 0) return 0; + + const setB = new Set(wordsB); + const intersection = wordsA.filter((w) => setB.has(w)); + return (2 * intersection.length) / (wordsA.length + wordsB.length); +} + +/** + * All-content-words gate: every significant word in `searchTitle` must + * appear in `candidateTitle`. Keeps "Children of Time" from accepting + * "Children of Ruin", which would otherwise share a high Dice score from + * the series stem. + * + * Containment is one-directional, so it accepts a candidate that adds words — + * "Dune" passes against "Dune Messiah". Anywhere a superset is the *wrong* + * book, gate on it in both directions. + */ +export function contentWordsMatch(searchTitle: string, candidateTitle: string): boolean { + const searchContent = contentWords(searchTitle); + if (searchContent.length === 0) return true; + const candidateContent = new Set(contentWords(candidateTitle)); + return searchContent.every((w) => candidateContent.has(w)); +} diff --git a/src/utils/bookMetadata/cbz.ts b/src/utils/bookMetadata/cbz.ts index 5309aafd..5759b5f9 100644 --- a/src/utils/bookMetadata/cbz.ts +++ b/src/utils/bookMetadata/cbz.ts @@ -4,6 +4,7 @@ import { unzipSync } from "fflate"; import type { BookCover, BookMetadata } from "./types"; import { extOf, mimeForExt } from "./shared"; +import { MAX_COVER_BYTES } from "./cover"; const IMAGE_EXTS = new Set(["jpg", "jpeg", "png", "gif", "bmp", "webp", "avif"]); @@ -16,20 +17,31 @@ const collator = new Intl.Collator(undefined, { numeric: true }); export function parseCbz(bytes: Uint8Array, fallbackTitle: string): BookMetadata { const fallback: BookMetadata = { title: fallbackTitle, authors: "" }; try { - const files = unzipSync(bytes, { - filter: (f) => IMAGE_EXTS.has(extOf(f.name)), + // Pass 1: index every page without inflating one. This used to decompress + // the entire comic — every page — in order to keep page 1. Returning + // `false` still walks the central directory, so the names (and sizes) are + // free; a 100 MB CBZ no longer inflates ~100 MB of pages it discards. + const pages: { name: string; originalSize: number }[] = []; + unzipSync(bytes, { + filter: (f) => { + if (IMAGE_EXTS.has(extOf(f.name))) { + pages.push({ name: f.name, originalSize: f.originalSize }); + } + return false; + }, }); - const names = Object.keys(files).sort((a, b) => collator.compare(a, b)); - if (names.length === 0) return fallback; + if (pages.length === 0) return fallback; - const first = names[0]; - if (!first) return fallback; - const ext = extOf(first) === "jpeg" ? "jpg" : extOf(first); - const cover: BookCover = { - bytes: files[first]!, - mime: mimeForExt(ext), - ext, - }; + pages.sort((a, b) => collator.compare(a.name, b.name)); + const first = pages[0]!; + if (first.originalSize > MAX_COVER_BYTES) return fallback; + + // Pass 2: inflate exactly the first page. + const data = unzipSync(bytes, { filter: (f) => f.name === first.name })[first.name]; + if (!data || data.length === 0) return fallback; + + const ext = extOf(first.name) === "jpeg" ? "jpg" : extOf(first.name); + const cover: BookCover = { bytes: data, mime: mimeForExt(ext), ext }; return { title: fallbackTitle, authors: "", cover }; } catch { return fallback; diff --git a/src/utils/bookMetadata/cover.ts b/src/utils/bookMetadata/cover.ts index 7707d9fc..96e8f288 100644 --- a/src/utils/bookMetadata/cover.ts +++ b/src/utils/bookMetadata/cover.ts @@ -8,6 +8,14 @@ /** Minimum width/height (px) for a cover to be considered "real". */ export const MIN_COVER_DIMENSION = 16; +/** + * Largest cover we will decompress out of an archive. The ZIP central + * directory tells us the decompressed size before we inflate anything, so this + * is checked ahead of the work rather than after it — a book advertising a + * 200 MB image as its cover simply doesn't get one. + */ +export const MAX_COVER_BYTES = 8 * 1024 * 1024; + /** * Decode the cover header with Bun's native image pipeline and confirm it is a * real, sensibly-sized image. Never throws — returns false on any failure. diff --git a/src/utils/bookMetadata/epub.ts b/src/utils/bookMetadata/epub.ts index 510f6f6d..948f41e9 100644 --- a/src/utils/bookMetadata/epub.ts +++ b/src/utils/bookMetadata/epub.ts @@ -1,6 +1,12 @@ import { unzipSync, strFromU8 } from "fflate"; import type { BookCover, BookMetadata } from "./types"; import { attr, decodeXmlEntities, mimeForExt } from "./shared"; +import { MAX_COVER_BYTES } from "./cover"; + +const IMAGE_NAME_RE = /\.(?:jpg|jpeg|png|gif|webp|svg)$/i; + +/** One archive entry we know about but have deliberately not inflated yet. */ +type ImageEntry = { name: string; originalSize: number }; /** Extract inner text of the first matching (or ) element. */ function firstDcValue(xml: string, tag: string): string | undefined { @@ -45,22 +51,25 @@ function resolveHref(opfPath: string, href: string): string { export function parseEpub(bytes: Uint8Array, fallbackTitle: string): BookMetadata { const fallback: BookMetadata = { title: fallbackTitle, authors: "" }; + // Pass 1: inflate ONLY container.xml and the .opf, while noting the name and + // size of every image without inflating any of them. + // + // This used to decompress every image in the archive in order to keep one. + // Returning `false` from the filter still walks the central directory — the + // entry's name and `originalSize` are available for free — so an index costs + // nothing and the ~100 MB of pages in a large CBZ-style EPUB is never + // materialised. See the second pass below for the one entry we do inflate. + const images: ImageEntry[] = []; let files: Record; try { - // Only decompress the small files we actually need plus any image (cover). files = unzipSync(bytes, { filter(file) { const n = file.name.toLowerCase(); - return ( - n === "meta-inf/container.xml" || - n.endsWith(".opf") || - n.endsWith(".jpg") || - n.endsWith(".jpeg") || - n.endsWith(".png") || - n.endsWith(".gif") || - n.endsWith(".webp") || - n.endsWith(".svg") - ); + if (IMAGE_NAME_RE.test(n)) { + images.push({ name: file.name, originalSize: file.originalSize }); + return false; + } + return n === "meta-inf/container.xml" || n.endsWith(".opf"); }, }); } catch { @@ -101,27 +110,43 @@ export function parseEpub(bytes: Uint8Array, fallbackTitle: string): BookMetadat const language = firstDcValue(opf, "language"); const identifier = firstDcValue(opf, "identifier"); - // 3. Locate the cover image. - const cover = findCover(opf, opfPath, byLowerName); + // 3. Locate the cover image, then inflate exactly that one entry. + const imagesByLowerName = new Map(); + for (const image of images) imagesByLowerName.set(image.name.toLowerCase(), image); + const cover = inflateCover(bytes, findCoverEntry(opf, opfPath, imagesByLowerName)); return { title, authors, language, identifier, cover }; } -function findCover( +/** + * Pass 2: decompress the single chosen image. Guarded on `originalSize` so a + * book advertising a 200 MB "cover" can't inflate unbounded — nothing we do + * with a cover needs more than MAX_COVER_BYTES. + */ +function inflateCover(bytes: Uint8Array, entry: ImageEntry | undefined): BookCover | undefined { + if (!entry || entry.originalSize > MAX_COVER_BYTES) return undefined; + let data: Uint8Array | undefined; + try { + data = unzipSync(bytes, { filter: (f) => f.name === entry.name })[entry.name]; + } catch { + return undefined; + } + if (!data || data.length === 0) return undefined; + const ext = (entry.name.split(".").pop() || "").toLowerCase(); + const normExt = ext === "jpeg" ? "jpg" : ext; + return { bytes: data, mime: mimeForExt(normExt), ext: normExt }; +} + +function findCoverEntry( opf: string, opfPath: string, - byLowerName: Map, -): BookCover | undefined { + byLowerName: Map, +): ImageEntry | undefined { const itemTags = opf.match(/]*>/gi) ?? []; - const findByHref = (href?: string): BookCover | undefined => { + const findByHref = (href?: string): ImageEntry | undefined => { if (!href) return undefined; - const resolved = resolveHref(opfPath, href).toLowerCase(); - const entry = byLowerName.get(resolved); - if (!entry) return undefined; - const ext = (entry.name.split(".").pop() || "").toLowerCase(); - const normExt = ext === "jpeg" ? "jpg" : ext; - return { bytes: entry.data, mime: mimeForExt(normExt), ext: normExt }; + return byLowerName.get(resolveHref(opfPath, href).toLowerCase()); }; // EPUB3: item with properties="cover-image". diff --git a/src/utils/bookMetadata/hash.ts b/src/utils/bookMetadata/hash.ts index a27ccb85..84b29f74 100644 --- a/src/utils/bookMetadata/hash.ts +++ b/src/utils/bookMetadata/hash.ts @@ -29,3 +29,29 @@ export function koreaderPartialMD5(bytes: Uint8Array): string { } return hasher.digest("hex"); } + +/** + * The same hash, read straight off a file instead of a buffer. + * + * The algorithm only ever touches twelve 1 KB windows, so it never needed the + * whole file resident — the buffer was incidental to how the upload path used + * to work. Reading the slices costs at most 12 KB regardless of file size, + * which is what lets an upload be hashed (and rejected as a duplicate) before + * anything is materialised. + * + * Must stay byte-for-byte equivalent to `koreaderPartialMD5`: the resulting + * hash is what lines an uploaded file up with the `document` id a KOReader + * device sends. `bookMetadata.test.ts` pins the two together. + */ +export async function koreaderPartialMD5File(file: Blob, size: number): Promise { + const step = 1024; + const window = 1024; + const hasher = new Bun.CryptoHasher("md5"); + for (let i = -1; i <= 10; i++) { + const offset = (step << ((2 * i) & 0x1f)) >>> 0; // i === -1 -> 0 + if (offset >= size) break; + const end = Math.min(offset + window, size); + hasher.update(new Uint8Array(await file.slice(offset, end).arrayBuffer())); + } + return hasher.digest("hex"); +} diff --git a/src/utils/bookMetadata/index.ts b/src/utils/bookMetadata/index.ts index ae7798b0..17f0d50c 100644 --- a/src/utils/bookMetadata/index.ts +++ b/src/utils/bookMetadata/index.ts @@ -96,10 +96,21 @@ export function detectFormat(bytes: Uint8Array, filename: string): FormatInfo { return unknown(ext); } -/** Parse metadata for any supported format. Never throws. */ -export function parseBook(bytes: Uint8Array, filename: string): BookMetadata { +/** + * Parse metadata for any supported format. Never throws. + * + * `formatInfo` is optional purely to save re-deriving it: the upload path + * already ran `detectFormat` against a 4 KB head to decide whether to accept + * the file at all, and re-running it here would be the only reason that call + * needed the full buffer. + */ +export function parseBook( + bytes: Uint8Array, + filename: string, + formatInfo?: FormatInfo, +): BookMetadata { const fallbackTitle = filename.replace(/\.[^.]+$/, ""); - const { format } = detectFormat(bytes, filename); + const { format } = formatInfo ?? detectFormat(bytes, filename); switch (format) { case "epub": return parseEpub(bytes, fallbackTitle); @@ -114,8 +125,8 @@ export function parseBook(bytes: Uint8Array, filename: string): BookMetadata { } } -export { koreaderPartialMD5 } from "./hash"; +export { koreaderPartialMD5, koreaderPartialMD5File } from "./hash"; export { looksLikeZip } from "./shared"; export { parseEpub } from "./epub"; -export { isUsableCover, MIN_COVER_DIMENSION } from "./cover"; +export { isUsableCover, MAX_COVER_BYTES, MIN_COVER_DIMENSION } from "./cover"; export type { BookCover, BookMetadata, EpubCover, EpubMetadata } from "./types"; diff --git a/src/utils/bookMetadata/testFixtures.ts b/src/utils/bookMetadata/testFixtures.ts new file mode 100644 index 00000000..649b8ba7 --- /dev/null +++ b/src/utils/bookMetadata/testFixtures.ts @@ -0,0 +1,148 @@ +/** + * Synthetic ebook fixtures for tests. + * + * Built with fflate's `zipSync` rather than checked in as binaries so the + * contents are readable in the diff and a test can vary one thing (a 1x1 cover, + * an oversized page) without a new blob appearing in the repo. + */ + +import { zipSync, strToU8 } from "fflate"; + +/** + * A real 32x32 PNG. Has to decode for `isUsableCover` — which reads the header + * with Bun's image pipeline — so it can't be arbitrary bytes. + */ +export const PNG_32 = (() => { + // 32x32, 8-bit RGBA, single IDAT of zlib-stored zeroes. + const raw = new Uint8Array(32 * (32 * 4 + 1)); // filter byte per scanline + const idat = Bun.deflateSync(raw); + const chunks: Uint8Array[] = []; + const be32 = (n: number) => + new Uint8Array([(n >>> 24) & 255, (n >>> 16) & 255, (n >>> 8) & 255, n & 255]); + const crcTable = (() => { + const t = new Uint32Array(256); + for (let n = 0; n < 256; n++) { + let c = n; + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + t[n] = c >>> 0; + } + return t; + })(); + const crc = (bytes: Uint8Array) => { + let c = 0xffffffff; + for (const b of bytes) c = crcTable[(c ^ b) & 0xff]! ^ (c >>> 8); + return (c ^ 0xffffffff) >>> 0; + }; + const chunk = (type: string, data: Uint8Array) => { + const typeBytes = new TextEncoder().encode(type); + const body = new Uint8Array(typeBytes.length + data.length); + body.set(typeBytes); + body.set(data, typeBytes.length); + chunks.push(be32(data.length), body, be32(crc(body))); + }; + const ihdr = new Uint8Array(13); + ihdr.set(be32(32), 0); + ihdr.set(be32(32), 4); + ihdr[8] = 8; // bit depth + ihdr[9] = 6; // RGBA + chunk("IHDR", ihdr); + chunk("IDAT", idat); + chunk("IEND", new Uint8Array(0)); + + const signature = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + const total = signature.length + chunks.reduce((n, c) => n + c.length, 0); + const out = new Uint8Array(total); + out.set(signature); + let offset = signature.length; + for (const c of chunks) { + out.set(c, offset); + offset += c.length; + } + return out; +})(); + +/** A 1x1 PNG — decodes fine, but below MIN_COVER_DIMENSION so it must be rejected. */ +export const PNG_1 = new Uint8Array([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, + 0x89, 0x00, 0x00, 0x00, 0x0a, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9c, 0x63, 0x00, 0x01, 0x00, 0x00, + 0x05, 0x00, 0x01, 0x0d, 0x0a, 0x2d, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, 0x44, 0xae, + 0x42, 0x60, 0x82, +]); + +export type EpubFixtureOptions = { + title?: string; + authors?: string[]; + language?: string; + /** Cover image bytes; omit for a book with no cover at all. */ + cover?: Uint8Array | undefined; + /** Extra padding, to push the file past a size threshold. */ + padBytes?: number; +}; + +/** A minimal but genuinely valid EPUB: mimetype, container.xml, OPF, cover. */ +export function makeEpub(options: EpubFixtureOptions = {}): Uint8Array { + const { + title = "The Test Book", + authors = ["A Test Author"], + language = "en", + cover = PNG_32, + padBytes = 0, + } = options; + + const manifest = cover + ? `` + : ""; + const opf = + `` + + `` + + `` + + `${title}` + + authors.map((a) => `${a}`).join("") + + `${language}` + + `${manifest}`; + + const files: Record = { + mimetype: strToU8("application/epub+zip"), + "META-INF/container.xml": strToU8( + `` + + ``, + ), + "OEBPS/content.opf": strToU8(opf), + }; + if (cover) files["OEBPS/images/cover.png"] = cover; + if (padBytes > 0) { + // Random so it doesn't deflate to nothing — the point of padding is size. + const pad = new Uint8Array(padBytes); + crypto.getRandomValues(pad); + files["OEBPS/pad.bin"] = pad; + } + return zipSync(files); +} + +/** + * A CBZ whose pages sort such that page 1 is small and page 2 is large — so a + * test can assert the cover extractor took the first page without inflating + * the rest. + */ +export function makeCbz(pageCount = 20, bigPageBytes = 200_000): Uint8Array { + const files: Record = { "001.png": PNG_32 }; + for (let i = 2; i <= pageCount; i++) { + const big = new Uint8Array(bigPageBytes); + big.set(PNG_32); + files[`${String(i).padStart(3, "0")}.png`] = big; + } + return zipSync(files); +} + +/** The smallest thing `detectFormat` accepts with no zip container. */ +export function makeFb2(title = "FB2 Book", author = "FB2 Author"): Uint8Array { + return strToU8( + `` + + `` + + `` + + `${title}` + + `${author.split(" ")[0]}${author.split(" ").slice(1).join(" ")}` + + ``, + ); +} diff --git a/src/utils/catalogBookService.test.ts b/src/utils/catalogBookService.test.ts new file mode 100644 index 00000000..e1381fe0 --- /dev/null +++ b/src/utils/catalogBookService.test.ts @@ -0,0 +1,83 @@ +/** + * Backfill progress persistence. + * + * `backfillCatalogBooks` runs for hours on the primary worker, but + * `/admin/backfill-catalog/progress` is answered by whichever of the three + * cluster workers the request lands on — and by a fresh process after any + * restart. Without the KV mirror those requests reported `idle` for a job that + * was running, or lost the outcome of one that had finished. + */ + +import { describe, it, expect, beforeEach } from "bun:test"; +import { createStorage } from "unstorage"; +import memoryDriver from "unstorage/drivers/memory"; +import type { Storage } from "unstorage"; + +import { getBackfillProgress, type BackfillProgress } from "./catalogBookService"; + +const KEY = "backfill:catalog_progress"; + +let kv: Storage; + +beforeEach(() => { + kv = createStorage({ driver: memoryDriver() }); +}); + +function stored(overrides: Partial): BackfillProgress { + return { + status: "running", + startedAt: "2026-08-01T00:00:00.000Z", + completedAt: null, + written: 120, + batches: 5, + totalPending: 900, + lastBatchAt: "2026-08-01T00:20:00.000Z", + error: null, + ...overrides, + }; +} + +describe("getBackfillProgress", () => { + it("reports idle when nothing has ever run", async () => { + expect((await getBackfillProgress(kv)).status).toBe("idle"); + }); + + it("still answers without a KV, for callers that have none", async () => { + expect((await getBackfillProgress()).status).toBe("idle"); + }); + + it("reads a completed run back out of the KV", async () => { + await kv.setItem(KEY, stored({ status: "completed", completedAt: "2026-08-01T01:00:00.000Z" })); + const progress = await getBackfillProgress(kv); + expect(progress.status).toBe("completed"); + expect(progress.written).toBe(120); + expect(progress.batches).toBe(5); + }); + + it("reports a stored 'running' as interrupted", async () => { + // A genuinely live run answers from this process's memory, so a *stored* + // "running" can only mean the process died mid-run. Leaving it as "running" + // would show a job that never finishes and never fails. + await kv.setItem(KEY, stored({ status: "running" })); + const progress = await getBackfillProgress(kv); + expect(progress.status).toBe("interrupted"); + expect(progress.completedAt).toBe("2026-08-01T00:20:00.000Z"); + expect(progress.error).toContain("restarted"); + }); + + it("falls back to idle rather than throwing on an unusable entry", async () => { + await kv.setItem(KEY, "{not json"); + expect((await getBackfillProgress(kv)).status).toBe("idle"); + }); + + it("survives the JSON round trip unstorage actually performs", async () => { + // The trap this pins: unstorage runs `destr` over what a driver returns, so + // persisting `JSON.stringify(progress)` reads back as an object and any + // `JSON.parse` of it throws — which silently discarded every stored run. + // Store the object; let unstorage handle serialization. + await kv.setItem(KEY, stored({ status: "completed" })); + const raw = await kv.getItem(KEY); + expect(typeof raw).toBe("object"); + expect((await getBackfillProgress(kv)).status).toBe("completed"); + }); +}); diff --git a/src/utils/catalogBookService.ts b/src/utils/catalogBookService.ts index a37e549e..6877a20f 100644 --- a/src/utils/catalogBookService.ts +++ b/src/utils/catalogBookService.ts @@ -2,6 +2,7 @@ import { Client } from "@atcute/client"; import { PasswordSession } from "@atcute/password-session"; import type { ActorIdentifier } from "@atcute/lexicons/syntax"; import type { Logger } from "pino"; +import type { Storage } from "unstorage"; import type { SessionClient } from "../auth/client"; import type { AppContext } from "../context"; import { createActorResolver } from "../bsky/id-resolver"; @@ -39,6 +40,8 @@ export async function createServiceAccountAgent( type CatalogCtx = Pick & { serviceAccountAgent: AppContext["serviceAccountAgent"] | undefined; + /** Optional: when present, backfill progress is persisted across restarts. */ + kv?: Storage; logger?: Logger; }; @@ -234,7 +237,7 @@ class RateLimitError extends Error { } export interface BackfillProgress { - status: "idle" | "running" | "completed" | "failed"; + status: "idle" | "running" | "completed" | "failed" | "interrupted"; startedAt: string | null; completedAt: string | null; written: number; @@ -244,6 +247,8 @@ export interface BackfillProgress { error: string | null; } +const BACKFILL_KV_KEY = "backfill:catalog_progress"; + let backfillProgress: BackfillProgress = { status: "idle", startedAt: null, @@ -255,8 +260,45 @@ let backfillProgress: BackfillProgress = { error: null, }; -export function getBackfillProgress(): BackfillProgress { - return { ...backfillProgress }; +/** + * Mirror the in-memory progress into the KV so it survives a restart. Fire and + * forget: this is observability for an admin endpoint, and failing to record it + * must never interrupt the backfill itself. + * + * Stores the object, not `JSON.stringify` of it. unstorage runs `destr` over + * whatever a driver returns, so a stored JSON *string* reads back as an object + * — round-tripping it through `JSON.parse` then throws, and the persisted + * progress is silently discarded. Same idiom as `enqueuePdsWrite`. + */ +function persistProgress(kv: Storage | undefined) { + if (!kv) return; + void kv.setItem(BACKFILL_KV_KEY, backfillProgress).catch(() => {}); +} + +/** + * Progress of the catalog backfill. Reads the in-memory value while a run is + * live in *this* process, and otherwise falls back to the KV — which is what + * makes the answer meaningful after a restart, and across the other cluster + * workers that never ran the backfill. + * + * A stored "running" status necessarily means the process died mid-run (a live + * run would have answered from memory), so it is reported as `interrupted` + * rather than left looking active forever. + */ +export async function getBackfillProgress(kv?: Storage): Promise { + if (backfillProgress.status !== "idle") return { ...backfillProgress }; + if (!kv) return { ...backfillProgress }; + const stored = await kv.getItem(BACKFILL_KV_KEY); + if (!stored || typeof stored !== "object" || typeof stored.status !== "string") { + return { ...backfillProgress }; + } + const parsed = { ...stored }; + if (parsed.status === "running") { + parsed.status = "interrupted"; + parsed.completedAt = parsed.lastBatchAt; + parsed.error = "Process restarted while backfill was running"; + } + return parsed; } /** @@ -306,6 +348,7 @@ export async function backfillCatalogBooks( lastBatchAt: null, error: null, }; + persistProgress(ctx.kv); try { while (true) { @@ -345,18 +388,21 @@ export async function backfillCatalogBooks( backfillProgress.written = written; backfillProgress.batches = batches; backfillProgress.lastBatchAt = new Date().toISOString(); + persistProgress(ctx.kv); await new Promise((r) => setTimeout(r, BATCH_DELAY_MS)); } backfillProgress.status = "completed"; backfillProgress.completedAt = new Date().toISOString(); + persistProgress(ctx.kv); wideEvent["outcome"] = "success"; return { written, batches }; } catch (err) { backfillProgress.status = "failed"; backfillProgress.completedAt = new Date().toISOString(); backfillProgress.error = err instanceof Error ? err.message : String(err); + persistProgress(ctx.kv); wideEvent["outcome"] = "error"; wideEvent["error"] = err instanceof Error ? { message: err.message, type: err.name } : String(err); diff --git a/src/utils/filenameMatching.test.ts b/src/utils/filenameMatching.test.ts new file mode 100644 index 00000000..1c801c61 --- /dev/null +++ b/src/utils/filenameMatching.test.ts @@ -0,0 +1,176 @@ +import { describe, it, expect } from "bun:test"; + +import { + authorsMatch, + filenameBasename, + filenameBookCandidates, + filenameKey, + koreaderFilenameHash, + normalizeAuthor, + normalizeTitle, + titlesEquivalent, +} from "./filenameMatching"; + +describe("koreaderFilenameHash", () => { + it("is md5 of the basename, matching KOSync's FILENAME checksum method", () => { + // KOReader: `md5(file_name)` where file_name is util.splitFilePathName's + // second return — the basename, extension included. This is the id such a + // client sends as `document`, so the value has to be byte-exact. + const expected = new Bun.CryptoHasher("md5").update("Dune.epub", "utf8").digest("hex"); + expect(koreaderFilenameHash("Dune.epub")).toBe(expected); + expect(koreaderFilenameHash("/mnt/onboard/books/Dune.epub")).toBe(expected); + expect(koreaderFilenameHash("D:\\books\\Dune.epub")).toBe(expected); + }); + + it("returns null rather than hashing nothing", () => { + expect(koreaderFilenameHash(null)).toBeNull(); + expect(koreaderFilenameHash("")).toBeNull(); + expect(koreaderFilenameHash(" ")).toBeNull(); + }); +}); + +describe("filenameBasename", () => { + it("handles both separators", () => { + expect(filenameBasename("/a/b/c.epub")).toBe("c.epub"); + expect(filenameBasename("a\\b\\c.epub")).toBe("c.epub"); + expect(filenameBasename("c.epub")).toBe("c.epub"); + }); +}); + +describe("filenameKey", () => { + it("survives the format conversion that broke the content hash", () => { + // The whole reason users switch off binary checksums: calibre re-encodes + // the file, so the bytes (and the extension) differ but the book does not. + expect(filenameKey("Dune - Frank Herbert.epub")).toBe(filenameKey("dune_-_frank_herbert.azw3")); + }); + + it("ignores release noise and copy markers", () => { + expect(filenameKey("Dune (z-lib.org) [Retail].epub")).toBe("dune"); + expect(filenameKey("Dune (1).epub")).toBe("dune"); + }); + + it("keeps a dot-number that is part of the title", () => { + // A generic `\.\w+$` strip would turn this into "foundation vol". + expect(filenameKey("Foundation Vol.2.epub")).toBe("foundation vol 2"); + }); + + it("folds diacritics so the same book from two sources agrees", () => { + expect(filenameKey("Les Misérables.epub")).toBe(filenameKey("Les Miserables.mobi")); + }); + + it("returns null when nothing indexable survives", () => { + // Must not be "", or every metadata-less document would match every other. + expect(filenameKey(null)).toBeNull(); + expect(filenameKey("---.epub")).toBeNull(); + }); +}); + +describe("filenameBookCandidates", () => { + const pairs = (filename: string) => + filenameBookCandidates(filename).map((c) => [c.title, c.authors]); + + it("emits both orderings of an A - B split", () => { + // Nothing in the string says which convention this is, and the caller + // resolves the ambiguity against the catalogue. + expect(pairs("Ursula K. Le Guin - The Dispossessed.epub")).toContainEqual([ + "The Dispossessed", + "Ursula K. Le Guin", + ]); + expect(pairs("The Dispossessed - Ursula K. Le Guin.epub")).toContainEqual([ + "The Dispossessed", + "Ursula K. Le Guin", + ]); + }); + + it("reads a trailing parenthetical as the author", () => { + expect(pairs("The Dispossessed (Ursula K. Le Guin).epub")).toContainEqual([ + "The Dispossessed", + "Ursula K. Le Guin", + ]); + }); + + it("drops a leading series index", () => { + expect(pairs("01 - The Fellowship of the Ring - J.R.R. Tolkien.epub")).toContainEqual([ + "The Fellowship of the Ring", + "J.R.R. Tolkien", + ]); + }); + + it("still offers a title for a bare filename", () => { + expect(pairs("The Dispossessed.epub")).toEqual([["The Dispossessed", null]]); + }); + + it("reads underscores as spaces", () => { + expect(pairs("Frank_Herbert_-_Dune.epub")).toContainEqual(["Dune", "Frank Herbert"]); + }); + + it("declines a two-character title, which is an index or a stray token", () => { + expect(filenameBookCandidates("a.epub")).toEqual([]); + expect(filenameBookCandidates("")).toEqual([]); + expect(filenameBookCandidates(null)).toEqual([]); + }); +}); + +describe("normalizeTitle", () => { + it("drops a trailing series or edition tail", () => { + expect(normalizeTitle("Dune (Dune Chronicles #1)")).toBe("dune"); + expect(normalizeTitle("Emma [Illustrated]")).toBe("emma"); + }); + + it("folds case, punctuation and diacritics", () => { + expect(normalizeTitle("The Hitchhiker's Guide")).toBe("the hitchhiker s guide"); + expect(normalizeTitle("Les Misérables")).toBe("les miserables"); + }); +}); + +describe("titlesEquivalent", () => { + it("ignores punctuation, stop words and word order", () => { + expect( + titlesEquivalent("Hitchhikers Guide to the Galaxy", "The Hitchhiker's Guide to the Galaxy"), + ).toBe(true); + expect(titlesEquivalent("Hobbit", "The Hobbit")).toBe(true); + }); + + it("rejects a title that merely contains the other", () => { + // The reason the word gate runs both ways: this is a real, different book + // by the same author, so no downstream check would catch it. + expect(titlesEquivalent("Dune", "Dune Messiah")).toBe(false); + expect(titlesEquivalent("Children of Time", "Children of Ruin")).toBe(false); + }); + + it("does not call two unmatched non-Latin titles equivalent", () => { + // Both sides normalize to no ASCII content words; that is absence of + // evidence, not agreement. + expect(titlesEquivalent("戦争と平和", "白鯨")).toBe(false); + expect(titlesEquivalent("戦争と平和", "戦争と平和")).toBe(true); + }); +}); + +describe("normalizeAuthor", () => { + it("un-inverts a Last, First name", () => { + expect(normalizeAuthor("Le Guin, Ursula K.")).toBe("ursula k le guin"); + }); +}); + +describe("authorsMatch", () => { + it("matches across punctuation and inversion", () => { + expect(authorsMatch("J.R.R. Tolkien", "J R R Tolkien")).toBe(true); + expect(authorsMatch("Tolkien, J.R.R.", "J.R.R. Tolkien")).toBe(true); + }); + + it("matches initials against the spelled-out name", () => { + expect(authorsMatch("J.R.R. Tolkien", "John Ronald Reuel Tolkien")).toBe(true); + }); + + it("does not accept a shared surname alone", () => { + // The reason the first-initial check exists: a surname on its own would + // link a book to the wrong member of a writing family. + expect(authorsMatch("Jane Tolkien", "John Tolkien")).toBe(false); + expect(authorsMatch("Tolkien", "John Tolkien")).toBe(false); + }); + + it("rejects unrelated names", () => { + expect(authorsMatch("Frank Herbert", "Ursula K. Le Guin")).toBe(false); + expect(authorsMatch("", "Frank Herbert")).toBe(false); + }); +}); diff --git a/src/utils/filenameMatching.ts b/src/utils/filenameMatching.ts new file mode 100644 index 00000000..91f95d4e --- /dev/null +++ b/src/utils/filenameMatching.ts @@ -0,0 +1,315 @@ +/** + * Filename-derived identity for e-reader documents. + * + * KOSync clients identify a document one of two ways (KOReader's + * `CHECKSUM_METHOD`, plugins/kosync.koplugin/main.lua): + * + * - `BINARY` (0, the default) — `md5` over 1 KiB samples of the file itself. + * That is `koreaderPartialMD5` in `bookMetadata/hash.ts`, and it is what + * `personal_book.contentHash` stores, which is why an uploaded file lines up + * with its synced progress by hash alone. + * - `FILENAME` (1) — plain `md5(basename)`, nothing to do with the bytes. + * Users switch to it precisely *because* their files are not byte-identical + * across devices (calibre conversion, image downscaling), so for those users + * the content hash can never match. `koreaderFilenameHash` reproduces it. + * + * Separately, KOReader's `send_metadata` option (and CrossPoint, which sends it + * unconditionally) puts the human-readable filename in the progress payload. A + * filename is often the *only* usable signal we get: plenty of documents arrive + * with no title/author metadata at all, and "Ursula K. Le Guin - The + * Dispossessed.epub" identifies a book perfectly well. + * + * Three derived values, used in that order of confidence: + * + * | value | matches | fuzzy? | + * | ----------------------- | ------------------------------------------ | ------ | + * | `koreaderFilenameHash` | a FILENAME-mode `sync_document.documentHash` | no | + * | `filenameKey` | another file's normalized name | a bit | + * | `filenameBookCandidates`| a `hive_book` title/author | yes | + */ + +import { contentWords } from "./bookMatching"; + +/** + * Extensions stripped before comparing two filenames. Deliberately a closed + * list rather than a `\.\w+$` regex: a generic strip mangles titles that end in + * a dot-number ("Foundation Vol.2", "Hitchhiker's 1.5"), and the whole point of + * the key is that two names for the same book collapse onto it. + * + * Conversion changes the extension — the .epub on the desktop is the .azw3 on + * the Kindle — so stripping it is what makes the key survive a calibre round + * trip at all. + */ +const EBOOK_EXTENSIONS = new Set([ + "epub", + "kepub", + "mobi", + "azw", + "azw3", + "azw4", + "prc", + "pdb", + "fb2", + "cbz", + "cbr", + "cb7", + "pdf", + "djvu", + "djv", + "txt", + "rtf", + "doc", + "docx", + "html", + "htm", + "xhtml", + "chm", + "lit", + "opf", + "zip", + "gz", +]); + +/** Parenthesised groups that are release noise, not part of the title. */ +const JUNK_PAREN = + /^(z-?lib(rary)?(\.org)?|libgen|anna'?s? archive|retail|repack|v\d+(\.\d+)*|\d+|epub|mobi|azw3?|pdf|scan|ocr|copy|dup(licate)?|final|fixed)$/i; + +const SEPARATORS = /\s+[-–—_]\s+/; + +export type FilenameCandidate = { + title: string; + /** null when the filename yielded no author signal at all. */ + authors: string | null; +}; + +/** Last path segment, tolerating both separators (KOReader runs on Windows too). */ +export function filenameBasename(path: string): string { + const cut = Math.max(path.lastIndexOf("/"), path.lastIndexOf("\\")); + return cut === -1 ? path : path.slice(cut + 1); +} + +/** + * `md5` of the basename — byte-for-byte what a KOSync client in FILENAME + * checksum mode sends as its `document` id. Matching this against + * `sync_document.documentHash` is exact, not fuzzy: it is the same protocol + * identifier, just computed on our side. + */ +export function koreaderFilenameHash(filename: string | null | undefined): string | null { + if (!filename) return null; + const name = filenameBasename(filename).trim(); + if (!name) return null; + return new Bun.CryptoHasher("md5").update(name, "utf8").digest("hex"); +} + +/** Strip a known ebook extension (including the `.fb2.zip` double form). */ +function stripExtension(name: string): string { + let out = name; + for (let i = 0; i < 2; i++) { + const dot = out.lastIndexOf("."); + if (dot <= 0) break; + const ext = out.slice(dot + 1).toLowerCase(); + if (!EBOOK_EXTENSIONS.has(ext)) break; + out = out.slice(0, dot); + } + return out; +} + +/** + * Everything both the key and the candidate parser want: basename, no + * extension, no `[...]` groups, no release-noise `(...)` groups, underscores + * read as spaces, whitespace collapsed. + */ +function cleanFilename(filename: string): string { + let name = stripExtension(filenameBasename(filename)); + name = name.replace(/\[[^\]]*\]/g, " "); + name = name.replace(/\(([^)]*)\)/g, (whole, inner: string) => + JUNK_PAREN.test(inner.trim()) ? " " : whole, + ); + name = name.replace(/_/g, " "); + return name.replace(/\s+/g, " ").trim(); +} + +/** + * Comparison key for "are these two files the same book". Folds case, + * punctuation and the extension away, so `Dune - Frank Herbert.epub` and + * `dune_-_frank_herbert.azw3` land on `dune frank herbert`. + * + * Returns null when nothing indexable survives, which callers must treat as + * "no key" rather than as a value to match on — otherwise every metadata-less + * document would match every other one. + */ +export function filenameKey(filename: string | null | undefined): string | null { + if (!filename) return null; + const key = cleanFilename(filename) + .normalize("NFKD") + .replace(/\p{M}+/gu, "") + .toLowerCase() + .replace(/[^\p{L}\p{N}]+/gu, " ") + .trim(); + return key || null; +} + +/** Diacritics folded away, so the ASCII-only helpers in `bookMatching` behave. */ +function foldDiacritics(input: string): string { + return input.normalize("NFKD").replace(/\p{M}+/gu, ""); +} + +/** Diacritics folded, series/edition tail dropped, punctuation still intact. */ +function titleBody(title: string): string { + // Tails: "Dune (Dune Chronicles #1)", "Emma [Illustrated]". + return foldDiacritics(title) + .toLowerCase() + .replace(/[([][^)\]]*[)\]]\s*$/g, "") + .trim(); +} + +/** Normalized form used to compare a filename-derived title to a `hive_book` one. */ +export function normalizeTitle(title: string): string { + return titleBody(title) + .replace(/[^\p{L}\p{N}]+/gu, " ") + .trim(); +} + +/** + * Whether two titles name the same book. + * + * Equal normalized strings, or the same set of significant words in any order + * — which buys tolerance for punctuation ("Hitchhiker's" / "Hitchhikers"), + * stop words ("The Hobbit" / "Hobbit") and ordering, all of which vary between + * a filename and a Goodreads title without changing the book. + * + * The word gate runs in **both** directions, unlike `contentWordsMatch`'s + * one-way containment. One-way is right for a CSV import, where the row has + * other identifiers to fall back on; here it would accept "Dune" against "Dune + * Messiah" — a real, different book, by the same author, so the author check + * downstream would wave it through and write the user's progress onto it. + */ +export function titlesEquivalent(a: string, b: string): boolean { + const na = normalizeTitle(a); + const nb = normalizeTitle(b); + if (!na || !nb) return false; + if (na === nb) return true; + + // From the punctuated body, not from `na`/`nb`: `normalizeTitle` turns + // punctuation into a space, which splits "hitchhiker's" into two words, while + // `contentWords` deletes it and yields the one word "hitchhikers". + const wa = contentWords(titleBody(a)); + const wb = contentWords(titleBody(b)); + // Empty on both sides means neither title survived ASCII normalization (a + // CJK or Cyrillic title, or one made entirely of stop words). That is not + // agreement — fall back to the string equality already tested above. + if (wa.length === 0 || wb.length === 0) return false; + if (wa.length !== wb.length) return false; + const sortedA = [...wa].sort(); + const sortedB = [...wb].sort(); + return sortedA.every((w, i) => w === sortedB[i]); +} + +/** Normalized form of a single personal name, with `Last, First` un-inverted. */ +export function normalizeAuthor(author: string): string { + let name = author.trim(); + const comma = name.indexOf(","); + if (comma > 0 && !name.slice(comma + 1).includes(",")) { + name = `${name.slice(comma + 1).trim()} ${name.slice(0, comma).trim()}`; + } + return name + .normalize("NFKD") + .replace(/\p{M}+/gu, "") + .toLowerCase() + .replace(/[^\p{L}\p{N}]+/gu, " ") + .trim(); +} + +/** + * Whether two author strings plausibly name the same person. + * + * Deliberately not a similarity score: each rule below is one concrete way the + * same author is written differently across a filename and Goodreads, and + * anything else is a miss. Initials are the interesting case — "J R R Tolkien" + * and "John Ronald Reuel Tolkien" agree on a surname and a first initial only. + * Comparing initials is therefore allowed, but only when one side actually *is* + * an initial: two spelled-out first names must be equal, or "Jane Tolkien" and + * "John Tolkien" would be the same person. + */ +export function authorsMatch(a: string, b: string): boolean { + const x = normalizeAuthor(a); + const y = normalizeAuthor(b); + if (!x || !y) return false; + if (x === y) return true; + + const xs = x.split(" "); + const ys = y.split(" "); + const xSurname = xs[xs.length - 1]; + const ySurname = ys[ys.length - 1]; + if (!xSurname || !ySurname || xSurname !== ySurname) return false; + // Single-token names ("Homer", "Plato") are already covered by equality; a + // surname-only match against a full name is too weak to accept. + if (xs.length === 1 || ys.length === 1) return false; + + const xFirst = xs[0]!; + const yFirst = ys[0]!; + if (xFirst === yFirst) return true; + if (xFirst.length > 1 && yFirst.length > 1) return false; + return xFirst[0] === yFirst[0]; +} + +function pushCandidate(out: FilenameCandidate[], title: string, authors: string | null): void { + const t = title.trim().replace(/^[-–—\s]+|[-–—\s]+$/g, ""); + const a = authors?.trim().replace(/^[-–—\s]+|[-–—\s]+$/g, "") || null; + // A "title" of one or two characters is a series index or a stray token, not + // something worth hashing against the whole catalogue. + if (t.length < 3) return; + if (a !== null && a.length < 2) return; + if (out.some((c) => c.title === t && c.authors === a)) return; + out.push({ title: t, authors: a }); +} + +/** + * Title/author guesses for a filename, most confident first. + * + * Both orderings of an `A - B` split are emitted, because both conventions are + * in the wild (calibre writes `Title - Author`, most torrents write `Author - + * Title`) and nothing in the string says which one this is. That is safe to do + * blindly *because* the caller resolves them against the catalogue: `hive_book.id` + * is a hash of title+author, so a wrong ordering simply hashes to an id that + * does not exist. Guessing costs a lookup, never a wrong link. + */ +export function filenameBookCandidates(filename: string | null | undefined): FilenameCandidate[] { + if (!filename) return []; + const cleaned = cleanFilename(filename); + if (!cleaned) return []; + + const out: FilenameCandidate[] = []; + + // "The Dispossessed (Ursula K. Le Guin)" — a trailing parenthetical that + // survived the junk filter is nearly always the author. + const trailingParen = cleaned.match(/^(.*?)\s*\(([^)]+)\)\s*$/); + const body = trailingParen ? trailingParen[1]!.trim() : cleaned; + if (trailingParen) pushCandidate(out, body, trailingParen[2]!); + + // Leading series index: "01 - The Fellowship of the Ring - J.R.R. Tolkien". + let parts = body + .split(SEPARATORS) + .map((p) => p.trim()) + .filter(Boolean); + if (parts.length > 1 && /^\d{1,3}\.?$/.test(parts[0]!)) parts = parts.slice(1); + + if (parts.length === 2) { + pushCandidate(out, parts[0]!, parts[1]!); + pushCandidate(out, parts[1]!, parts[0]!); + } else if (parts.length > 2) { + const head = parts[0]!; + const tail = parts[parts.length - 1]!; + const middle = parts.slice(1).join(" - "); + const front = parts.slice(0, -1).join(" - "); + // Either end can be the author; the rest is the title. + pushCandidate(out, middle, head); + pushCandidate(out, front, tail); + } + + // Whole name as a title, with whatever author signal we found. Last resort, + // but it is the only candidate for a bare "The Dispossessed.epub". + pushCandidate(out, parts.join(" - "), null); + + return out; +} diff --git a/src/utils/personalLibrary.ts b/src/utils/personalLibrary.ts index d447c759..27734431 100644 --- a/src/utils/personalLibrary.ts +++ b/src/utils/personalLibrary.ts @@ -8,16 +8,30 @@ import type { Database } from "../db"; export const OPDS_PAGE_SIZE = 24; /** - * Largest accepted ebook upload. Every upload path materialises the whole file - * as a `Uint8Array` (the KOReader partial MD5 and the format parsers both need - * random access), so this is a direct per-request ceiling on native memory — - * enforce it against the *declared* size before reading the body. + * Largest accepted ebook upload. The body itself streams to disk, but the + * format parsers need the whole file in one contiguous buffer (fflate reads a + * ZIP's central directory from the end), so this remains the per-request + * ceiling on native memory for the parse step. Enforce it against the + * *declared* size first, then again while streaming. */ export const MAX_PERSONAL_BOOK_BYTES = 100 * 1024 * 1024; -/** Root directory for all personal library files, adjacent to the DB. */ +/** + * Root directory for all personal library files. `LIBRARY_DIR` wins when set, + * so the library can live on a different volume from the DB; otherwise it sits + * adjacent to the DB as it always has. + */ export function getLibraryDir(): string { - return path.join(path.dirname(env.DB_PATH), "library"); + return env.LIBRARY_DIR || path.join(path.dirname(env.DB_PATH), "library"); +} + +/** + * Scratch directory for in-flight uploads. Deliberately under the library root + * so the finished file can be `rename`d into place rather than copied — that + * only holds within one filesystem. + */ +export function getLibraryTmpDir(): string { + return path.join(getLibraryDir(), ".tmp"); } /** Directory for a specific book: `{libraryDir}/{did}/{contentHash}/` */ @@ -50,6 +64,29 @@ export async function removeUserDir(did: string): Promise { await rm(path.join(getLibraryDir(), did), { recursive: true, force: true }); } +/** Total bytes one user may store across their personal library. */ +export function getStorageQuota(): number { + return env.PERSONAL_LIBRARY_QUOTA_BYTES; +} + +/** + * Bytes this user currently stores. Derived with a `SUM`, never a maintained + * counter: the quota itself bounds the row count (2 GB over a ~3 MB median + * epub is ~700 rows), `idx_personal_book_user_size` makes it an index-only + * scan, and a derived total cannot drift. A counter would need a backfill, + * decrements in both delete paths, and a repair job — and `removeBookDir` is + * best-effort, so a failed `rm` after a row delete would leave the counter + * under-reporting forever while the disk quietly filled. + */ +export async function getStorageUsage(db: Database, userDid: string): Promise { + const row = await db + .selectFrom("personal_book") + .select((eb) => eb.fn.coalesce(eb.fn.sum("sizeBytes"), eb.lit(0)).as("used")) + .where("userDid", "=", userDid) + .executeTakeFirst(); + return Number(row?.used ?? 0); +} + export type PersonalBookDownload = | { notModified: true; headers: Record } | { notModified: false; stream: ReadableStream; headers: Record }; @@ -59,7 +96,7 @@ export type PersonalBookDownload = * and `*` forms, and tolerates the `W/` prefix a client may echo back — the * validator is a content hash, so a weak match is still the same bytes. */ -function etagMatches(ifNoneMatch: string | null | undefined, etag: string): boolean { +export function etagMatches(ifNoneMatch: string | null | undefined, etag: string): boolean { if (!ifNoneMatch) return false; const normalize = (raw: string) => raw.trim().replace(/^W\//, ""); return ifNoneMatch diff --git a/src/utils/syncMatching.test.ts b/src/utils/syncMatching.test.ts new file mode 100644 index 00000000..c17602ef --- /dev/null +++ b/src/utils/syncMatching.test.ts @@ -0,0 +1,351 @@ +import { describe, it, expect, beforeEach, afterEach } from "bun:test"; +import { Database as DatabaseSync } from "bun:sqlite"; +import { Kysely, SqliteDialect } from "kysely"; + +import { wrapBunSqliteForKysely } from "../bun-sqlite-kysely"; +import { migrateToLatest, type Database, type DatabaseSchema } from "../db"; +import { getHiveId } from "../scrapers/getHiveId"; +import type { HiveId } from "../types"; +import { filenameKey, koreaderFilenameHash } from "./filenameMatching"; +import { matchSyncDocument, matchSyncDocumentForUser } from "./syncMatching"; + +const DID = "did:plc:testuser"; + +/** Hand-built like the other DB suites; `createDb` reads a mocked `env`. */ +async function createTestDb(): Promise { + const sqlite = new DatabaseSync(":memory:"); + sqlite.exec("PRAGMA journal_mode = WAL"); + const db = new Kysely({ + dialect: new SqliteDialect({ database: wrapBunSqliteForKysely(sqlite) }), + }); + await migrateToLatest(db, sqlite); + return db; +} + +describe("matchSyncDocument", () => { + let db: Database; + + const insert = async (title: string, authors: string, ratingsCount = 0): Promise => { + const id = getHiveId({ title, authors }); + await db + .insertInto("hive_book") + .values({ + id: id as never, + title, + rawTitle: title, + authors, + ratingsCount, + source: "goodreads", + thumbnail: "", + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } as never) + .execute(); + return id; + }; + + beforeEach(async () => { + db = await createTestDb(); + }); + + afterEach(async () => { + await db.destroy(); + }); + + describe("tier 1 — exact hash of the client's metadata", () => { + it("matches title + authors", async () => { + const id = await insert("The Dispossessed", "Ursula K. Le Guin"); + expect( + await matchSyncDocument(db, { title: "The Dispossessed", authors: "Ursula K. Le Guin" }), + ).toBe(id); + }); + + it("returns null when nothing identifies the document", async () => { + await insert("The Dispossessed", "Ursula K. Le Guin"); + expect(await matchSyncDocument(db, {})).toBeNull(); + }); + + it("splits newline-separated authors, which is how KOReader sends them", async () => { + // `metadata.authors` is `doc_props.authors`, one of the props KOReader + // edits with `allow_newline = true`. Treated as one name it matches + // nobody. + const id = await insert("Good Omens", "Terry Pratchett", 900); + expect( + await matchSyncDocument(db, { + title: "Good Omens", + authors: "Neil Gaiman\nTerry Pratchett", + filename: "Good Omens.epub", + }), + ).toBe(id); + }); + }); + + describe("tier 2 — exact hash of filename-derived pairs", () => { + it("matches a document that carries only a filename", async () => { + // The case this whole path exists for: KOSync metadata is optional, and + // a client that sends just the filename used to be unmatchable. + const id = await insert("The Dispossessed", "Ursula K. Le Guin"); + expect( + await matchSyncDocument(db, { + filename: "Ursula K. Le Guin - The Dispossessed.epub", + }), + ).toBe(id); + }); + + it("matches regardless of which side of the dash the author is on", async () => { + const id = await insert("The Dispossessed", "Ursula K. Le Guin"); + expect( + await matchSyncDocument(db, { filename: "The Dispossessed - Ursula K. Le Guin.epub" }), + ).toBe(id); + }); + + it("crosses a client title with a filename author", async () => { + const id = await insert("Dune", "Frank Herbert"); + expect( + await matchSyncDocument(db, { title: "Dune", filename: "Frank Herbert - Dune.epub" }), + ).toBe(id); + }); + + it("parses the client's title, which KOReader derives from the filename", async () => { + // `display_title` is `props.title or splitFileNameType(filepath)`, so a + // document with no embedded title sends the filename stem as its title. + const id = await insert("The Dispossessed", "Ursula K. Le Guin"); + expect(await matchSyncDocument(db, { title: "Ursula K. Le Guin - The Dispossessed" })).toBe( + id, + ); + }); + + it("prefers the client's own metadata over a filename guess", async () => { + const real = await insert("Dune", "Frank Herbert"); + await insert("Dune", "Someone Else"); + expect( + await matchSyncDocument(db, { + title: "Dune", + authors: "Frank Herbert", + filename: "Someone Else - Dune.epub", + }), + ).toBe(real); + }); + }); + + describe("tier 3 — fuzzy, filename only", () => { + it("matches a title the id hash could not, when the author agrees", async () => { + // The filename title is exact but the author is written differently, so + // the title+author hash misses entirely. + const id = await insert("The Dispossessed", "Ursula K. Le Guin", 100); + expect( + await matchSyncDocument(db, { filename: "Le Guin, Ursula - The Dispossessed.epub" }), + ).toBe(id); + }); + + it("accepts an unambiguous title with no author anywhere", async () => { + const id = await insert("The Dispossessed", "Ursula K. Le Guin", 100); + expect(await matchSyncDocument(db, { filename: "The Dispossessed.epub" })).toBe(id); + }); + + it("refuses to pick between books that share a title", async () => { + // Ranking is by popularity, which says nothing about which one this is. + await insert("Dune", "Frank Herbert", 900); + await insert("Dune", "A Different Author", 5); + expect(await matchSyncDocument(db, { filename: "Dune.epub" })).toBeNull(); + }); + + it("refuses a title match whose author disagrees", async () => { + await insert("Dune", "Frank Herbert", 900); + expect(await matchSyncDocument(db, { filename: "Ursula K. Le Guin - Dune.epub" })).toBeNull(); + }); + + it("does not match a book that merely ranks first for the filename", async () => { + await insert("The Girl with the Dragon Tattoo", "Stieg Larsson", 100); + expect(await matchSyncDocument(db, { filename: "The Girl.epub" })).toBeNull(); + }); + + it("tolerates punctuation and stop-word differences in the title", async () => { + const id = await insert("The Hitchhiker's Guide to the Galaxy", "Douglas Adams", 900); + expect( + await matchSyncDocument(db, { + filename: "Douglas Adams - Hitchhikers Guide to the Galaxy.epub", + }), + ).toBe(id); + }); + + it("does not accept a sequel that merely contains the title", async () => { + // "Dune" is a content-word subset of "Dune Messiah" and the author agrees, + // so one-directional containment would link the user's progress to the + // wrong book. A different book by the same author is the easiest way to + // get this wrong and the hardest for the user to notice. + await insert("Dune Messiah", "Frank Herbert", 900); + expect(await matchSyncDocument(db, { filename: "Frank Herbert - Dune.epub" })).toBeNull(); + }); + + it("ignores a series tail on the catalogue title", async () => { + const id = await insert("Dune (Dune Chronicles #1)", "Frank Herbert", 900); + expect(await matchSyncDocument(db, { filename: "Frank Herbert - Dune.epub" })).toBe(id); + }); + }); +}); + +describe("matchSyncDocumentForUser", () => { + let db: Database; + + const insert = async (title: string, authors: string, ratingsCount = 0): Promise => { + const id = getHiveId({ title, authors }); + await db + .insertInto("hive_book") + .values({ + id: id as never, + title, + rawTitle: title, + authors, + ratingsCount, + source: "goodreads", + thumbnail: "", + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } as never) + .execute(); + return id; + }; + + const upload = async (opts: { + contentHash: string; + filename: string; + title: string; + authors?: string | null; + hiveId?: HiveId | null; + }) => { + const now = new Date().toISOString(); + await db + .insertInto("personal_book") + .values({ + userDid: DID, + contentHash: opts.contentHash, + hiveId: opts.hiveId ?? null, + filename: opts.filename, + filenameHash: koreaderFilenameHash(opts.filename), + filenameKey: filenameKey(opts.filename), + title: opts.title, + authors: opts.authors ?? null, + language: null, + format: "epub", + mime: "application/epub+zip", + filePath: "/tmp/x.epub", + coverPath: null, + coverMime: null, + sizeBytes: 1, + createdAt: now, + updatedAt: now, + }) + .execute(); + }; + + beforeEach(async () => { + db = await createTestDb(); + }); + + afterEach(async () => { + await db.destroy(); + }); + + it("matches a default-configured client, which sends only a content hash", async () => { + // KOReader defaults: checksum_method BINARY, send_metadata off. The whole + // request identifies the book as one partial-MD5 and nothing else, so + // matching the payload alone can never work — but that hash is the + // contentHash of a file whose metadata we parsed at upload time. + const id = await insert("The Dispossessed", "Ursula K. Le Guin"); + await upload({ + contentHash: "partial-md5", + filename: "book.epub", + title: "The Dispossessed", + authors: "Ursula K. Le Guin", + }); + + expect(await matchSyncDocument(db, { title: null, authors: null, filename: null })).toBeNull(); + expect(await matchSyncDocumentForUser(db, DID, { documentHash: "partial-md5" })).toBe(id); + }); + + it("inherits a link the user already established on the file", async () => { + const id = await insert("Dune", "Frank Herbert"); + await upload({ + contentHash: "partial-md5", + filename: "book.epub", + title: "Something Unmatchable", + hiveId: id, + }); + + expect(await matchSyncDocumentForUser(db, DID, { documentHash: "partial-md5" })).toBe(id); + }); + + it("writes the resolved book back onto the file and marks it owned", async () => { + const id = await insert("The Dispossessed", "Ursula K. Le Guin"); + await upload({ + contentHash: "partial-md5", + filename: "book.epub", + title: "The Dispossessed", + authors: "Ursula K. Le Guin", + }); + const now = new Date().toISOString(); + await db + .insertInto("user_book") + .values({ + uri: "at://x/1", + cid: "c", + userDid: DID, + createdAt: now, + indexedAt: now, + hiveId: id, + title: "The Dispossessed", + authors: "Ursula K. Le Guin", + owned: 0, + } as never) + .execute(); + + await matchSyncDocumentForUser(db, DID, { documentHash: "partial-md5" }); + + const file = await db.selectFrom("personal_book").select("hiveId").executeTakeFirstOrThrow(); + expect(file.hiveId).toBe(id); + const book = await db.selectFrom("user_book").select("owned").executeTakeFirstOrThrow(); + expect(book.owned).toBe(1); + }); + + it("splits comma-separated authors, which is how uploads store them", async () => { + // `parseBook` joins epub dc:creator values with ", ". + const id = await insert("Good Omens", "Terry Pratchett", 900); + await upload({ + contentHash: "partial-md5", + filename: "Good Omens.epub", + title: "Good Omens", + authors: "Neil Gaiman, Terry Pratchett", + }); + + expect(await matchSyncDocumentForUser(db, DID, { documentHash: "partial-md5" })).toBe(id); + }); + + it("still un-inverts a single Last, First author", async () => { + // The same comma that separates two authors also inverts one name, so both + // readings are tried rather than guessed between. + const id = await insert("The Dispossessed", "Ursula K. Le Guin", 900); + await upload({ + contentHash: "partial-md5", + filename: "x.epub", + title: "The Dispossessed", + authors: "Le Guin, Ursula K.", + }); + + expect(await matchSyncDocumentForUser(db, DID, { documentHash: "partial-md5" })).toBe(id); + }); + + it("does not reach another user's uploads", async () => { + await insert("The Dispossessed", "Ursula K. Le Guin"); + await upload({ + contentHash: "partial-md5", + filename: "book.epub", + title: "The Dispossessed", + authors: "Ursula K. Le Guin", + }); + + expect( + await matchSyncDocumentForUser(db, "did:plc:someoneelse", { documentHash: "partial-md5" }), + ).toBeNull(); + }); +}); diff --git a/src/utils/syncMatching.ts b/src/utils/syncMatching.ts index 7f6a582f..1cc22a60 100644 --- a/src/utils/syncMatching.ts +++ b/src/utils/syncMatching.ts @@ -1,6 +1,19 @@ +import { sql } from "kysely"; import type { Database } from "../db"; import type { HiveId } from "../types"; import { getHiveId } from "../scrapers/getHiveId"; +import { parseAuthors } from "./authorMatching"; +import { ftsMatchQuery } from "./ftsQuery"; +import { + authorsMatch, + filenameBookCandidates, + filenameKey, + normalizeAuthor, + normalizeTitle, + titlesEquivalent, + type FilenameCandidate, +} from "./filenameMatching"; +import { similarityScore } from "./bookMatching"; /** * Sentinel written to `sync_document.hiveId` when the user asserts a synced @@ -14,27 +27,324 @@ import { getHiveId } from "../scrapers/getHiveId"; */ export const NO_HIVE_MATCH = "bk_none" as HiveId; +/** + * Cap on how much filename guessing one progress push is allowed to pay for. + * This runs on every push for a document that has not matched yet, so an + * unmatchable document re-pays it every few minutes, per device. + */ +const MAX_FTS_QUERIES = 4; +const FTS_LIMIT = 50; + +/** + * "This synced document and this uploaded file are the same book." + * + * Three ways that can be true, and a KOSync client only ever gives us one of + * them, so all three have to be tried: + * + * 1. `contentHash = documentHash` — the client is in BINARY checksum mode and + * the file we hold is byte-identical to the one on the device. The original + * (and only) rule. + * 2. `filenameHash = documentHash` — the client is in FILENAME mode, so what it + * calls a document id is md5 of the basename. Nothing about the bytes is + * involved and rule 1 can never fire for these users. + * 3. `filenameKey = filenameKey` — neither hash lines up, but the client sent a + * readable filename that normalizes to the same thing as ours. This is the + * calibre-conversion case: same book, different bytes *and* a different + * extension. + * + * Written as raw SQL because both directions of the relationship need it, as a + * correlated subquery rather than a join — a document can match more than one + * file and vice versa, and a join would fan those out into duplicate rows in + * the library grid and break its pagination. + * + * Both `filenameKey` columns are nullable and `NULL = NULL` is not true in SQL, + * so a document with no filename cannot match a file with no key. + */ +export const SAME_BOOK_FILE = sql`( + personal_book.contentHash = sync_document.documentHash + OR personal_book.filenameHash = sync_document.documentHash + OR personal_book.filenameKey = sync_document.filenameKey +)`; + +type FtsRow = { id: HiveId; title: string; authors: string | null; ratingsCount: number | null }; + +/** + * Split an author *signal* into individual names. Three sources reach this and + * each separates authors differently: + * + * - KOReader's `metadata.authors` is **newline**-separated (`doc_props.authors` + * is one of the three props its metadata editor opens with + * `allow_newline = true`). + * - `personal_book.authors` is **comma**-separated (`parseBook` joins epub + * `dc:creator` values with ", "). + * - `hive_book.authors` is tab-separated, but that side goes through + * `parseAuthors`, not here. + * + * The comma is ambiguous — it also inverts a single name ("Le Guin, Ursula") — + * so rather than guess, the whole string is emitted *alongside* the split + * parts and both interpretations are tried. That is safe because signals are + * only ever used as corroborating evidence: one that matches nothing simply + * fails to confirm, it cannot select a book on its own. + */ +function splitAuthorSignal(value: string | null | undefined): string[] { + if (!value) return []; + const parts = value + .split(/[\r\n\t;&]|\band\b/i) + .map((a) => a.trim()) + .filter(Boolean); + const withCommaSplits = parts.flatMap((part) => + part.includes(",") ? [part, ...part.split(",").map((a) => a.trim())] : [part], + ); + return dedupe(withCommaSplits.filter(Boolean)); +} + +function dedupe(values: string[]): string[] { + return [...new Set(values)]; +} + +/** Merge candidate lists, keeping the first (most confident) of each pair. */ +function dedupeCandidates(candidates: FilenameCandidate[]): FilenameCandidate[] { + const seen = new Set(); + return candidates.filter((c) => { + const key = `${c.title}\0${c.authors ?? ""}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +/** + * Resolve a synced e-reader document to a `hive_book`. + * + * Three tiers, strongest first. The rule that governs all of them is that a + * wrong link is worse than no link — it writes someone's reading progress onto + * a book they aren't reading, and (via `bridgeProgressToUserBook`) mirrors that + * to their PDS. A miss just leaves the document unlinked for the user to + * connect by hand, and the progress itself is stored either way, so e-reader + * sync is unaffected. + * + * 1. **Exact id hash of the supplied metadata.** `hive_book.id` is a hash of + * the lowercased title + author, so a hit is an exact identity, not a + * search. + * 2. **Exact id hash of title/author pairs parsed out of the filename**, + * including pairs that cross client metadata with a filename-derived author. + * Still exact: a wrong guess hashes to an id that does not exist. + * 3. **FTS on the filename-derived title**, accepted only when the normalized + * title is *equal* (not merely ranked first) and the author agrees. With no + * author signal at all, only an unambiguous single-book title is accepted. + * + * Tiers 2 and 3 exist because the filename is frequently the only thing we get. + * KOSync's own metadata is optional (KOReader's `send_metadata` defaults off; + * CrossPoint sends it) and plenty of documents carry no embedded title or + * author at all — but "Ursula K. Le Guin - The Dispossessed.epub" names a book + * perfectly well. + * + * This takes only what the client sent. Prefer `matchSyncDocumentForUser`, + * which also brings the user's own uploaded files to bear — it is the only + * thing that helps a default-configured client, which sends no metadata at all. + */ export async function matchSyncDocument( db: Database, metadata: { title?: string | null; authors?: string | null; filename?: string | null }, ): Promise { - const { title, authors } = metadata; - if (!title) return null; - - const authorStr = authors || "Unknown"; - - // Exact match only: the derived HiveId is a hash of the normalized - // title + author. A confident hit auto-bridges to the user's book; anything - // else is left unlinked for the user to connect manually, so we never write - // progress to the wrong book. The sync_document row (and its progress) is - // stored regardless, so e-reader sync itself is unaffected by a miss. - const candidateId = getHiveId({ title, authors: authorStr }); - const exact = await db - .selectFrom("hive_book") - .select("id") - .where("id", "=", candidateId) - .executeTakeFirst(); - if (exact) return exact.id; + const { title, authors, filename } = metadata; + + // The client's `title` may itself be a filename. KOReader sends + // `doc_props.display_title`, which is `props.title or + // splitFileNameType(filepath)` — for any document with no embedded title (a + // conversion, most scanned PDFs, plenty of epubs) that is literally the + // filename minus its extension, dashes and all. So parse it the same way. + const candidates = dedupeCandidates([ + ...filenameBookCandidates(filename), + ...filenameBookCandidates(title), + ]); + if (!title && candidates.length === 0) return null; + + // ── Tiers 1 + 2: exact id hashes, most confident first ── + const ids: HiveId[] = []; + const considerId = (t: string, a: string) => { + const id = getHiveId({ title: t, authors: a }); + if (!ids.includes(id)) ids.push(id); + }; + + if (title) considerId(title, authors || "Unknown"); + for (const c of candidates) { + if (c.authors) considerId(c.title, c.authors); + if (authors) considerId(c.title, authors); + } + // The client named the book but not the author; the filename may have one. + if (title) { + for (const c of candidates) { + if (c.authors) considerId(title, c.authors); + } + } + + if (ids.length > 0) { + const found = await db.selectFrom("hive_book").select("id").where("id", "in", ids).execute(); + if (found.length > 0) { + const hit = new Set(found.map((r) => r.id)); + // Resolve in the order the ids were generated, so client metadata beats a + // filename guess and a two-sided guess beats a one-sided one. + const best = ids.find((id) => hit.has(id)); + if (best) return best; + } + } + + // ── Tier 3: fuzzy, on the filename only ── + const authorSignals = dedupe([ + ...splitAuthorSignal(authors), + ...candidates.flatMap((c) => splitAuthorSignal(c.authors)), + ]); + + // Candidate pool. `hive_book_fts` matches phrases, so searching it for the + // title only finds books whose title tokenizes the same way — "Hitchhikers + // Guide" never reaches "The Hitchhiker's Guide". Searching for the *author* + // instead sidesteps that: an author's name is spelled the same either way, + // and their handful of books can then be compared on title in JS, where the + // comparison can be as forgiving as it needs to be. The title search stays + // as the only option when the filename yields no author at all. + const queries: string[] = []; + for (const signal of authorSignals) { + const q = ftsMatchQuery(normalizeAuthor(signal)); + if (q) queries.push(q); + } + const seenTitles = new Set(); + for (const candidate of candidates) { + const want = normalizeTitle(candidate.title); + if (!want || seenTitles.has(want)) continue; + seenTitles.add(want); + const q = ftsMatchQuery(candidate.title); + if (q) queries.push(q); + } + + const pool = new Map(); + for (const match of dedupe(queries).slice(0, MAX_FTS_QUERIES)) { + const rows = ( + await sql` + SELECT b.id, b.title, b.authors, b.ratingsCount + FROM hive_book_fts f + JOIN hive_book b ON b.rowid = f.rowid + WHERE hive_book_fts MATCH ${match} + ORDER BY b.ratingsCount DESC, b.rating DESC + LIMIT ${FTS_LIMIT} + `.execute(db) + ).rows; + for (const row of rows) if (!pool.has(row.id)) pool.set(row.id, row); + } + if (pool.size === 0) return null; + + const books = [...pool.values()].sort((a, b) => (b.ratingsCount ?? 0) - (a.ratingsCount ?? 0)); + + for (const candidate of candidates) { + // Ranking is by popularity, which says nothing about whether the top hit is + // *this* book. Only titles that name the same book are eligible. + const eligible = books.filter((r) => titlesEquivalent(candidate.title, r.title)); + if (eligible.length === 0) continue; + + if (authorSignals.length > 0) { + const byAuthor = eligible.filter((r) => + parseAuthors(r.authors || "").some((bookAuthor) => + authorSignals.some((signal) => authorsMatch(bookAuthor, signal)), + ), + ); + // Several editions can agree on both; prefer the closest title, then the + // popularity order the query already applied. + const hit = byAuthor.reduce( + (best, r) => + best === null || + similarityScore(candidate.title, r.title) > similarityScore(candidate.title, best.title) + ? r + : best, + null, + ); + if (hit) return hit.id; + // The title matched but no author did: this is a different book with the + // same name. Fall through to the next candidate rather than to the + // no-author rule below, which would accept it. + continue; + } + + // No author anywhere. Accept only a title that names exactly one book in + // the catalogue — otherwise we would be picking the most popular of several + // unrelated books that happen to share a title. + const distinct = new Set(eligible.map((r) => r.id)); + if (distinct.size === 1) return eligible[0]!.id; + } return null; } + +/** + * Resolve a synced document to a book using everything we hold for this user, + * not just what the client sent. + * + * This is the entry point the KOSync routes use, and it exists for the + * **default** KOReader configuration, which is the majority: `checksum_method` + * is BINARY and `send_metadata` is off, so the entire request identifies the + * book as one partial-MD5 hash and nothing else. `matchSyncDocument` has + * nothing to work with — no title, no author, no filename — and returns null + * every time, forever, no matter how good its tiers get. + * + * But that hash *is* `personal_book.contentHash`. If the user has uploaded the + * file, we already parsed real title/author metadata out of the ebook itself at + * upload time, and may already have resolved it to a book. So: find the file + * first, inherit its book if it has one, and otherwise match on the file's + * metadata. The upload path already pushes a link the other way when the + * document exists first (`uploadPersonalBook` step 9); this closes the opposite + * ordering, where the file is uploaded before the e-reader ever syncs it. + */ +export async function matchSyncDocumentForUser( + db: Database, + userDid: string, + doc: { + documentHash: string; + filename?: string | null; + title?: string | null; + authors?: string | null; + }, +): Promise { + const docFilenameKey = filenameKey(doc.filename); + const file = await db + .selectFrom("personal_book") + .select(["id", "hiveId", "title", "authors", "filename"]) + .where("userDid", "=", userDid) + .where((eb) => + eb.or([ + eb("contentHash", "=", doc.documentHash), + eb("filenameHash", "=", doc.documentHash), + ...(docFilenameKey ? [eb("filenameKey", "=", docFilenameKey)] : []), + ]), + ) + // A byte-identical file is a stronger claim than a same-name one. + .orderBy(sql`CASE WHEN contentHash = ${doc.documentHash} THEN 0 ELSE 1 END`, "asc") + .executeTakeFirst(); + + if (file?.hiveId && file.hiveId !== NO_HIVE_MATCH) return file.hiveId; + + let hiveId = await matchSyncDocument(db, doc); + if (!hiveId && file) { + // The ebook's own metadata, parsed from the file at upload time. Usually + // better than anything a filename can offer, and for a default-configured + // client it is the only thing there is. + hiveId = await matchSyncDocument(db, { + title: file.title, + authors: file.authors, + filename: file.filename, + }); + } + + if (hiveId && file && !file.hiveId) { + // Keep the file and the document agreeing, and mirror the upload path's + // "you own a copy" flag. + await db.updateTable("personal_book").set({ hiveId }).where("id", "=", file.id).execute(); + await db + .updateTable("user_book") + .set({ owned: 1 }) + .where("userDid", "=", userDid) + .where("hiveId", "=", hiveId) + .where("owned", "=", 0) + .execute(); + } + + return hiveId; +} diff --git a/src/utils/uploadPersonalBook.test.ts b/src/utils/uploadPersonalBook.test.ts new file mode 100644 index 00000000..0b8c9557 --- /dev/null +++ b/src/utils/uploadPersonalBook.test.ts @@ -0,0 +1,592 @@ +import { describe, it, expect, beforeEach, afterEach } from "bun:test"; +import { Database as DatabaseSync } from "bun:sqlite"; +import { Kysely, SqliteDialect } from "kysely"; +import { createStorage } from "unstorage"; +import memoryDriver from "unstorage/drivers/memory"; +import { readdir, rm } from "node:fs/promises"; +import path from "node:path"; +import type { Storage } from "unstorage"; + +import { wrapBunSqliteForKysely } from "../bun-sqlite-kysely"; +import { migrateToLatest, type DatabaseSchema, type Database } from "../db"; +import type { HiveId } from "../types"; +import { koreaderPartialMD5 } from "./bookMetadata/index"; +import { makeCbz, makeEpub, makeFb2, PNG_1 } from "./bookMetadata/testFixtures"; +import { filenameKey, koreaderFilenameHash } from "./filenameMatching"; +import { getHiveId } from "../scrapers/getHiveId"; +import { NO_HIVE_MATCH } from "./syncMatching"; +import { + bookFilePath, + coverFilePath, + ensureDir, + getLibraryTmpDir, + getStorageQuota, + getStorageUsage, + personalBookDir, + MAX_PERSONAL_BOOK_BYTES, +} from "./personalLibrary"; +import { uploadPersonalBook, type UploadPersonalBookResult } from "./uploadPersonalBook"; + +const DID = "did:plc:testuser"; +const OTHER_DID = "did:plc:someoneelse"; +const HIVE_A = "bk_aaaaaaaa" as HiveId; + +let db: Database; +let kv: Storage; + +async function createTestDb(): Promise { + const sqlite = new DatabaseSync(":memory:"); + sqlite.exec("PRAGMA journal_mode = WAL"); + const database = new Kysely({ + dialect: new SqliteDialect({ database: wrapBunSqliteForKysely(sqlite) }), + }); + await migrateToLatest(database, sqlite); + return database; +} + +/** Upload from a buffer, through the same stream path a real request uses. */ +function upload( + bytes: Uint8Array, + filename: string, + opts: { userDid?: string; declaredLength?: number | undefined; chunked?: boolean } = {}, +): Promise { + const userDid = opts.userDid ?? DID; + const body = new ReadableStream({ + start(controller) { + // Two chunks, so the streaming path is genuinely exercised rather than + // degenerating into a single write. + const mid = Math.floor(bytes.length / 2); + controller.enqueue(bytes.subarray(0, mid)); + controller.enqueue(bytes.subarray(mid)); + controller.close(); + }, + }); + return uploadPersonalBook({ + db, + kv, + userDid, + filename, + // `chunked` models a request with no Content-Length. + source: { + kind: "stream", + body, + declaredLength: opts.chunked ? undefined : (opts.declaredLength ?? bytes.length), + }, + }); +} + +async function seedSyncDocument(opts: { + documentHash: string; + hiveId?: HiveId | null; + filename?: string | null; + title?: string | null; + authors?: string | null; + percentage?: number; + userDid?: string; +}): Promise { + await db + .insertInto("sync_document") + .values({ + userDid: opts.userDid ?? DID, + provider: "kosync", + documentHash: opts.documentHash, + hiveId: opts.hiveId ?? null, + filename: opts.filename ?? null, + filenameKey: filenameKey(opts.filename ?? null), + title: opts.title ?? null, + authors: opts.authors ?? null, + progressData: JSON.stringify({ + progress: "1", + percentage: opts.percentage ?? 0.5, + device: "kindle", + device_id: "d1", + timestamp: 1, + }), + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + }) + .execute(); +} + +/** + * Seed a catalog book. Returns its id, which is derived from title+author — + * `matchSyncDocument`'s strongest tier recomputes exactly that hash and looks + * it up, so a fuzzy-match test only works with the real derived id. + */ +async function seedHiveBook(title: string, authors: string): Promise { + const id = getHiveId({ title, authors }); + await db + .insertInto("hive_book") + .values({ + id, + title, + rawTitle: title, + authors, + source: "goodreads", + thumbnail: "", + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + } as never) + .execute(); + return id; +} + +async function seedUserBook(hiveId: HiveId, owned = 0, userDid = DID): Promise { + await db + .insertInto("user_book") + .values({ + uri: `at://${userDid}/buzz.bookhive.book/${hiveId}`, + cid: "cid", + userDid, + hiveId, + title: "t", + authors: "a", + status: "buzz.bookhive.defs#reading", + owned, + createdAt: "2026-08-01T00:00:00.000Z", + indexedAt: "2026-08-01T00:00:00.000Z", + }) + .execute(); +} + +/** Seed a row directly, for quota arithmetic that shouldn't touch the disk. */ +async function seedPersonalBook(contentHash: string, sizeBytes: number, userDid = DID) { + await db + .insertInto("personal_book") + .values({ + userDid, + contentHash, + filename: `${contentHash}.epub`, + title: "Seeded", + format: "epub", + mime: "application/epub+zip", + filePath: `/tmp/${contentHash}.epub`, + sizeBytes, + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + }) + .execute(); +} + +async function tmpEntries(): Promise { + try { + return (await readdir(getLibraryTmpDir())).filter((n) => n.endsWith(".part")); + } catch { + return []; + } +} + +beforeEach(async () => { + db = await createTestDb(); + kv = createStorage({ driver: memoryDriver() }); + // The preload points LIBRARY_DIR at a per-pid tmp root; make sure it exists + // and is empty so leftover files can't make an assertion pass. + await ensureDir(getLibraryTmpDir()); + await rm(personalBookDir(DID, ""), { recursive: true, force: true }).catch(() => {}); +}); + +afterEach(async () => { + for (const did of [DID, OTHER_DID]) { + await rm(path.dirname(personalBookDir(did, "x")), { recursive: true, force: true }).catch( + () => {}, + ); + } + for (const name of await tmpEntries()) { + await rm(path.join(getLibraryTmpDir(), name), { force: true }); + } +}); + +describe("uploadPersonalBook — happy path", () => { + it("writes the file and cover to disk and persists every column", async () => { + const bytes = makeEpub({ title: "Dune", authors: ["Frank Herbert"], language: "en" }); + const result = await upload(bytes, "Dune.epub"); + + expect(result.ok).toBe(true); + if (!result.ok) return; + + expect(result.book.title).toBe("Dune"); + expect(result.book.authors).toBe("Frank Herbert"); + expect(result.book.language).toBe("en"); + expect(result.book.format).toBe("epub"); + expect(result.book.mime).toBe("application/epub+zip"); + expect(result.book.sizeBytes).toBe(bytes.length); + expect(result.book.coverUrl).toBe(`/library/covers/${result.book.contentHash}`); + + const hash = result.book.contentHash; + const stored = Bun.file(bookFilePath(DID, hash, "epub")); + expect(await stored.exists()).toBe(true); + expect(stored.size).toBe(bytes.length); + + const cover = Bun.file(coverFilePath(DID, hash, "png")); + expect(await cover.exists()).toBe(true); + + const row = await db + .selectFrom("personal_book") + .selectAll() + .where("userDid", "=", DID) + .executeTakeFirstOrThrow(); + expect(row.title).toBe("Dune"); + expect(row.authors).toBe("Frank Herbert"); + expect(row.language).toBe("en"); + expect(row.sizeBytes).toBe(bytes.length); + expect(row.filename).toBe("Dune.epub"); + expect(row.filenameHash).toBe(koreaderFilenameHash("Dune.epub")); + expect(row.filenameKey).toBe(filenameKey("Dune.epub")); + expect(row.coverMime).toBe("image/png"); + expect(row.coverPath).toBe(coverFilePath(DID, hash, "png")); + }); + + it("computes the same content hash streaming as in memory", async () => { + // The KOReader-compat invariant: this hash is what lines an uploaded file + // up with the `document` id a device sends. If the streamed and in-memory + // implementations ever diverge, every FILENAME/BINARY match silently stops. + for (const [bytes, name] of [ + [makeEpub(), "a.epub"], + [makeFb2(), "b.fb2"], + [makeEpub({ padBytes: 70_000 }), "c.epub"], + ] as const) { + const result = await upload(bytes, name); + expect(result.ok).toBe(true); + if (result.ok) expect(result.book.contentHash).toBe(koreaderPartialMD5(bytes)); + await db.deleteFrom("personal_book").execute(); + } + }); + + it("leaves no temp file behind", async () => { + await upload(makeEpub(), "x.epub"); + expect(await tmpEntries()).toEqual([]); + }); + + it("reports storage usage after the upload", async () => { + const bytes = makeEpub(); + const result = await upload(bytes, "x.epub"); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.storageUsedBytes).toBe(bytes.length); + expect(result.storageQuotaBytes).toBe(getStorageQuota()); + }); +}); + +describe("uploadPersonalBook — cover handling", () => { + it("stores no cover when the extracted image is too small to be real", async () => { + const result = await upload(makeEpub({ cover: PNG_1 }), "tiny.epub"); + expect(result.ok).toBe(true); + if (!result.ok) return; + + expect(result.book.coverUrl).toBeUndefined(); + const row = await db + .selectFrom("personal_book") + .select(["coverPath", "coverMime"]) + .executeTakeFirstOrThrow(); + expect(row.coverPath).toBeNull(); + expect(row.coverMime).toBeNull(); + + // And nothing was written for it. + const dir = await readdir(personalBookDir(DID, result.book.contentHash)); + expect(dir.some((n) => n.startsWith("cover."))).toBe(false); + }); + + it("takes the first page of a CBZ as the cover", async () => { + const result = await upload(makeCbz(), "comic.cbz"); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.book.format).toBe("cbz"); + expect(await Bun.file(coverFilePath(DID, result.book.contentHash, "png")).exists()).toBe(true); + }); +}); + +describe("uploadPersonalBook — empty metadata normalisation", () => { + it("stores NULL rather than empty string for missing authors and language", async () => { + // A CBZ carries no metadata at all, so parseBook returns authors: "". + const result = await upload(makeCbz(2, 100), "Nameless.cbz"); + expect(result.ok).toBe(true); + if (!result.ok) return; + + expect(result.book.authors).toBeUndefined(); + expect(result.book.language).toBeUndefined(); + + const row = await db + .selectFrom("personal_book") + .select(["authors", "language"]) + .executeTakeFirstOrThrow(); + expect(row.authors).toBeNull(); + expect(row.language).toBeNull(); + + // The point of NULL over "": this predicate has to find the row. + const missing = await db + .selectFrom("personal_book") + .select("id") + .where("authors", "is", null) + .execute(); + expect(missing).toHaveLength(1); + }); +}); + +describe("uploadPersonalBook — sync document linking", () => { + it("links via an exact content hash", async () => { + const bytes = makeEpub(); + await seedSyncDocument({ documentHash: koreaderPartialMD5(bytes), hiveId: HIVE_A }); + + const result = await upload(bytes, "x.epub"); + expect(result.ok).toBe(true); + if (result.ok) expect(result.book.hiveId).toBe(HIVE_A); + }); + + it("links via the filename hash a FILENAME-mode client sends", async () => { + await seedSyncDocument({ + documentHash: koreaderFilenameHash("Dune.epub")!, + hiveId: HIVE_A, + }); + + const result = await upload(makeEpub(), "Dune.epub"); + expect(result.ok).toBe(true); + if (result.ok) expect(result.book.hiveId).toBe(HIVE_A); + }); + + it("links via the normalised filename key across a format conversion", async () => { + await seedSyncDocument({ + documentHash: "unrelated-hash", + hiveId: HIVE_A, + filename: "Dune.azw3", + }); + + const result = await upload(makeEpub(), "Dune.epub"); + expect(result.ok).toBe(true); + if (result.ok) expect(result.book.hiveId).toBe(HIVE_A); + }); + + it("prefers a byte-exact document over a fuzzy title match", async () => { + // The regression this pins: the XRPC path used to run the fuzzy matcher + // first, so a title guess could beat an exact documentHash. Both links are + // available here and the exact one has to win. + const bytes = makeEpub({ title: "Dune", authors: ["Frank Herbert"] }); + const fuzzyId = await seedHiveBook("Dune", "Frank Herbert"); + await seedSyncDocument({ documentHash: koreaderPartialMD5(bytes), hiveId: HIVE_A }); + + const result = await upload(bytes, "Dune.epub"); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.book.hiveId).toBe(HIVE_A); + expect(result.book.hiveId).not.toBe(fuzzyId); + } + }); + + it("falls back to fuzzy matching when no document matches", async () => { + const hiveId = await seedHiveBook("Dune", "Frank Herbert"); + const result = await upload( + makeEpub({ title: "Dune", authors: ["Frank Herbert"] }), + "Dune.epub", + ); + expect(result.ok).toBe(true); + if (result.ok) expect(result.book.hiveId).toBe(hiveId); + }); + + it("never adopts the dismissal sentinel", async () => { + const bytes = makeEpub({ title: "Untraceable Xyzzy", authors: ["Nobody At All"] }); + await seedSyncDocument({ + documentHash: koreaderPartialMD5(bytes), + hiveId: NO_HIVE_MATCH, + }); + + const result = await upload(bytes, "x.epub"); + expect(result.ok).toBe(true); + if (result.ok) expect(result.book.hiveId).toBeUndefined(); + }); + + it("writes the link back onto an unmatched document and bridges its progress", async () => { + const bytes = makeEpub({ title: "Dune", authors: ["Frank Herbert"] }); + const hiveId = await seedHiveBook("Dune", "Frank Herbert"); + await seedUserBook(hiveId); + // A document the device has been pushing progress for that never matched. + await seedSyncDocument({ + documentHash: koreaderPartialMD5(bytes), + hiveId: null, + percentage: 0.42, + }); + + const result = await upload(bytes, "Dune.epub"); + expect(result.ok).toBe(true); + + const doc = await db.selectFrom("sync_document").select("hiveId").executeTakeFirstOrThrow(); + expect(doc.hiveId).toBe(hiveId); + + // The percentage it had already recorded is now on the user's book, rather + // than sitting unused until the device next syncs. + const userBook = await db + .selectFrom("user_book") + .select(["bookProgress", "owned"]) + .executeTakeFirstOrThrow(); + expect(JSON.parse(userBook.bookProgress!).percent).toBe(42); + expect(userBook.owned).toBe(1); + }); + + it("does not clobber a document the user linked by hand", async () => { + const bytes = makeEpub({ title: "Dune", authors: ["Frank Herbert"] }); + await seedHiveBook("Dune", "Frank Herbert"); + await seedSyncDocument({ documentHash: koreaderPartialMD5(bytes), hiveId: HIVE_A }); + + await upload(bytes, "Dune.epub"); + const doc = await db.selectFrom("sync_document").select("hiveId").executeTakeFirstOrThrow(); + expect(doc.hiveId).toBe(HIVE_A); + }); + + it("flips owned only for the uploading user", async () => { + const bytes = makeEpub(); + await seedSyncDocument({ documentHash: koreaderPartialMD5(bytes), hiveId: HIVE_A }); + await seedUserBook(HIVE_A, 0, DID); + await seedUserBook(HIVE_A, 0, OTHER_DID); + + await upload(bytes, "x.epub"); + + const mine = await db + .selectFrom("user_book") + .select("owned") + .where("userDid", "=", DID) + .executeTakeFirstOrThrow(); + const theirs = await db + .selectFrom("user_book") + .select("owned") + .where("userDid", "=", OTHER_DID) + .executeTakeFirstOrThrow(); + expect(mine.owned).toBe(1); + expect(theirs.owned).toBe(0); + }); +}); + +describe("uploadPersonalBook — rejections", () => { + it("rejects an unsupported format and leaves nothing behind", async () => { + const result = await upload(new TextEncoder().encode("just some text"), "notes.txt"); + expect(result).toEqual({ ok: false, reason: "unsupported-format", filename: "notes.txt" }); + expect(await db.selectFrom("personal_book").selectAll().execute()).toHaveLength(0); + expect(await tmpEntries()).toEqual([]); + }); + + it("rejects an empty body", async () => { + const result = await upload(new Uint8Array(0), "empty.epub"); + expect(result).toEqual({ ok: false, reason: "empty" }); + expect(await tmpEntries()).toEqual([]); + }); + + it("rejects a file merely named .epub", async () => { + // detectFormat validates the extension's claim against the magic bytes — + // this is the real gate, not the declared Content-Type. + const result = await upload(new TextEncoder().encode("not a zip at all"), "fake.epub"); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("unsupported-format"); + }); + + it("rejects a duplicate without writing anything", async () => { + const bytes = makeEpub(); + const first = await upload(bytes, "x.epub"); + expect(first.ok).toBe(true); + + const second = await upload(bytes, "x.epub"); + expect(second.ok).toBe(false); + if (!second.ok) expect(second.reason).toBe("duplicate"); + + expect(await db.selectFrom("personal_book").selectAll().execute()).toHaveLength(1); + expect(await tmpEntries()).toEqual([]); + }); + + it("lets two users hold the same bytes independently", async () => { + // The duplicate check is scoped to (userDid, contentHash) — a cross-user + // hash collision is the normal case (the same book), not an error. + const bytes = makeEpub(); + const mine = await upload(bytes, "x.epub", { userDid: DID }); + const theirs = await upload(bytes, "x.epub", { userDid: OTHER_DID }); + + expect(mine.ok).toBe(true); + expect(theirs.ok).toBe(true); + expect(await db.selectFrom("personal_book").selectAll().execute()).toHaveLength(2); + if (mine.ok && theirs.ok) { + expect(await Bun.file(bookFilePath(DID, mine.book.contentHash, "epub")).exists()).toBe(true); + expect( + await Bun.file(bookFilePath(OTHER_DID, theirs.book.contentHash, "epub")).exists(), + ).toBe(true); + } + }); + + it("caps a chunked body with no declared length", async () => { + // The case hono's bodyLimit() buffered whole: no Content-Length, so the + // ceiling can only be enforced while streaming. + const oversized = new ReadableStream({ + start(controller) { + const chunk = new Uint8Array(1024 * 1024); + for (let sent = 0; sent <= MAX_PERSONAL_BOOK_BYTES; sent += chunk.length) { + controller.enqueue(chunk); + } + controller.close(); + }, + }); + + const result = await uploadPersonalBook({ + db, + kv, + userDid: DID, + filename: "huge.epub", + source: { kind: "stream", body: oversized, declaredLength: undefined }, + }); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("too-large"); + expect(await tmpEntries()).toEqual([]); + }, 60_000); +}); + +describe("uploadPersonalBook — storage quota", () => { + it("refuses an upload that would cross the quota, before reading the body", async () => { + const bytes = makeEpub(); + // One byte of headroom short of what this upload needs. + const used = getStorageQuota() - bytes.length + 1; + await seedPersonalBook("seeded", used); + + const result = await upload(bytes, "x.epub"); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.reason).toBe("quota-exceeded"); + if (result.reason === "quota-exceeded") { + expect(result.quotaBytes).toBe(getStorageQuota()); + expect(result.usedBytes).toBe(used); + } + } + // Nothing stored, and the body was never drained to disk. + expect(await db.selectFrom("personal_book").selectAll().execute()).toHaveLength(1); + expect(await tmpEntries()).toEqual([]); + }); + + it("enforces the quota against the real size when none was declared", async () => { + const bytes = makeEpub(); + await seedPersonalBook("seeded", getStorageQuota() - Math.floor(bytes.length / 2)); + + const result = await upload(bytes, "x.epub", { chunked: true }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("quota-exceeded"); + expect(await db.selectFrom("personal_book").selectAll().execute()).toHaveLength(1); + expect(await tmpEntries()).toEqual([]); + }); + + it("allows an upload that lands exactly on the quota", async () => { + const bytes = makeEpub(); + await seedPersonalBook("seeded", getStorageQuota() - bytes.length); + + const result = await upload(bytes, "x.epub"); + expect(result.ok).toBe(true); + }); + + it("does not count another user's books toward mine", async () => { + await seedPersonalBook("theirs", getStorageQuota(), OTHER_DID); + const result = await upload(makeEpub(), "x.epub"); + expect(result.ok).toBe(true); + expect(await getStorageUsage(db, OTHER_DID)).toBe(getStorageQuota()); + }); + + it("frees space when a book is deleted", async () => { + const bytes = makeEpub(); + const first = await upload(bytes, "x.epub"); + expect(first.ok).toBe(true); + expect(await getStorageUsage(db, DID)).toBe(bytes.length); + + await db.deleteFrom("personal_book").where("userDid", "=", DID).execute(); + expect(await getStorageUsage(db, DID)).toBe(0); + }); +}); diff --git a/src/utils/uploadPersonalBook.ts b/src/utils/uploadPersonalBook.ts new file mode 100644 index 00000000..736b1ece --- /dev/null +++ b/src/utils/uploadPersonalBook.ts @@ -0,0 +1,535 @@ +/** + * The one implementation of "put this ebook in this user's library". + * + * There used to be two: `POST /library/upload` (live) and `processBookUpload` + * in the XRPC router (dead code that claimed in its own doc comment to be the + * shared core). They drifted — different cover validation, different sync + * matching, only one of them writing the link back onto `sync_document` — which + * is exactly the failure mode a "shared" helper nobody shares is supposed to + * prevent. Both routes are now thin adapters over this function. + * + * The ordering of the pipeline below is the design, not an accident. Two + * properties it exists to hold: + * + * - **Nothing large is resident unless we are actually going to keep it.** The + * body streams to a temp file, bounded by the sink's 1 MB high-water mark + * regardless of how big the upload is; format detection reads a 4 KB + * head; the KOReader hash reads twelve 1 KB windows; the duplicate check + * happens before the parse. A rejected upload — wrong format, too big, over + * quota, already present — never allocates a copy of the file. Only + * `parseBook` needs the whole thing, and that step is behind a semaphore. + * - **The row commits before the bytes move into place.** The quota is + * evaluated inside the INSERT, so a rejected upload unlinks a temp file + * rather than discovering the problem after writing 100 MB to its final home. + * + * Errors are a discriminated result, never a throw. `processBookUpload` threw + * `XRPCError` from a util, which meant a Hono route had to catch an HTTP-shaped + * exception and translate it back. Each adapter now owns its own status codes. + */ + +import path from "node:path"; +import { rename, rm, readdir, stat } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { sql } from "kysely"; +import type { Storage } from "unstorage"; + +import type { Database } from "../db"; +import type { HiveId } from "../types"; +import { env } from "../env"; +import { Semaphore, SemaphoreFullError, SemaphoreTimeoutError } from "./semaphore"; +import { + detectFormat, + isUsableCover, + koreaderPartialMD5File, + parseBook, + type BookMetadata, + type FormatInfo, +} from "./bookMetadata/index"; +import { + bookFilePath, + coverFilePath, + ensureDir, + getLibraryTmpDir, + getStorageQuota, + getStorageUsage, + personalBookDir, + MAX_PERSONAL_BOOK_BYTES, +} from "./personalLibrary"; +import { matchSyncDocument, NO_HIVE_MATCH } from "./syncMatching"; +import { bridgeProgressToUserBook } from "./syncBridge"; +import { filenameKey, koreaderFilenameHash } from "./filenameMatching"; +import type { SyncProgressData } from "../types"; + +/** + * Bytes needed for format detection. `detectFormat` reads at most the first 512 + * bytes (the FictionBook sniff), the first 4 for the ZIP magic, and 60..68 for + * the MOBI magic — 4 KB is generous and keeps one read. + */ +const FORMAT_HEAD_BYTES = 4096; + +/** + * The parse is the only step holding a whole file (<=100 MB) in native memory, + * so this is the memory bound on uploads. It is **per process** — with + * `WEB_CONCURRENCY=4` the cluster-wide ceiling is `limit x 4 x 100 MB`, so 2 + * here means roughly 840 MB worst case rather than the previous unbounded + * ~300 MB *per in-flight upload*. + * + * `maxPending` sheds load instead of queueing waiters: each queued caller holds + * its closure — and therefore its temp file handle — alive, and a client that + * gets a fast 503 retries better than one that hangs. + */ +const parseSemaphore = new Semaphore(env.UPLOAD_PARSE_CONCURRENCY, { + label: "ebook-parse", + maxPending: 16, + acquireTimeoutMs: 30_000, +}); + +export type UploadSource = + | { kind: "stream"; body: ReadableStream; declaredLength?: number | undefined } + | { kind: "bytes"; bytes: Uint8Array }; + +/** Exactly `buzz.bookhive.getPersonalLibrary#personalBookView`. */ +export type PersonalBookView = { + contentHash: string; + title: string; + authors?: string | undefined; + language?: string | undefined; + format: string; + mime: string; + sizeBytes: number; + createdAt: string; + updatedAt: string; + hiveId?: string | undefined; + coverUrl?: string | undefined; +}; + +export type UploadPersonalBookResult = + | { ok: true; book: PersonalBookView; storageUsedBytes: number; storageQuotaBytes: number } + | { ok: false; reason: "empty" } + | { ok: false; reason: "too-large"; limitBytes: number } + | { ok: false; reason: "unsupported-format"; filename: string } + | { ok: false; reason: "duplicate"; contentHash: string } + | { + ok: false; + reason: "quota-exceeded"; + usedBytes: number; + quotaBytes: number; + fileBytes: number; + } + | { ok: false; reason: "busy" }; + +/** Reasons in the order a caller is likely to want them, for exhaustive maps. */ +export type UploadFailureReason = Extract["reason"]; + +class TooLargeError extends Error {} + +/** + * Stream a body to disk with a hard byte ceiling, holding at most one buffer's + * worth of it at a time. + * + * This is what replaces hono's `bodyLimit()` on the multipart route. That + * middleware only short-circuits on `Content-Length`; with a chunked body it + * drains the entire stream into an array and rebuilds the Request, so a + * compliant 100 MB chunked upload was buffered there *and again* by + * `formData()`. Capping while writing bounds every path identically. + */ +async function writeCapped(dest: string, source: UploadSource, cap: number): Promise { + // `highWaterMark` is the real memory bound here: the sink buffers up to this + // much before flushing to disk, and awaiting each write is the backpressure + // signal. (Bun's FileSink returns a number synchronously today, but the type + // allows a Promise — awaiting handles both and costs nothing.) + const sink = Bun.file(dest).writer({ highWaterMark: 1024 * 1024 }); + let written = 0; + try { + if (source.kind === "bytes") { + if (source.bytes.length > cap) throw new TooLargeError(); + await sink.write(source.bytes); + written = source.bytes.length; + } else { + const reader = source.body.getReader(); + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + written += value.length; + // Checked before the write, so nothing past the cap ever reaches disk. + if (written > cap) throw new TooLargeError(); + await sink.write(value); + } + } finally { + await reader.cancel().catch(() => {}); + } + } + await sink.end(); + } catch (err) { + // Close the fd rather than leaking it; the caller unlinks the file. + try { + await sink.end(); + } catch { + /* already ended */ + } + throw err; + } + return written; +} + +/** + * Insert the row only if it keeps the user under quota. + * + * The `SUM` is evaluated *inside* the statement rather than read first and + * compared in JS. SQLite serialises writers, so this is exact: two concurrent + * uploads cannot both observe the pre-insert total. A per-process mutex would + * not have worked anyway — production runs four independent processes against + * one file (`server/cluster.ts`). + * + * Returns false when the quota rejected the row. + */ +async function insertIfUnderQuota( + db: Database, + row: { + userDid: string; + contentHash: string; + hiveId: HiveId | null; + filename: string; + filenameHash: string | null; + filenameKey: string | null; + title: string; + authors: string | null; + language: string | null; + format: string; + mime: string; + filePath: string; + coverPath: string | null; + coverMime: string | null; + sizeBytes: number; + createdAt: string; + updatedAt: string; + }, + quotaBytes: number, +): Promise { + const result = await sql` + INSERT INTO personal_book + (userDid, contentHash, hiveId, filename, filenameHash, filenameKey, title, authors, + language, format, mime, filePath, coverPath, coverMime, sizeBytes, createdAt, updatedAt) + SELECT ${row.userDid}, ${row.contentHash}, ${row.hiveId}, ${row.filename}, + ${row.filenameHash}, ${row.filenameKey}, ${row.title}, ${row.authors}, + ${row.language}, ${row.format}, ${row.mime}, ${row.filePath}, + ${row.coverPath}, ${row.coverMime}, ${row.sizeBytes}, ${row.createdAt}, + ${row.updatedAt} + WHERE ( + SELECT COALESCE(SUM(sizeBytes), 0) FROM personal_book WHERE userDid = ${row.userDid} + ) + ${row.sizeBytes} <= ${quotaBytes} + `.execute(db); + return (result.numAffectedRows ?? 0n) > 0n; +} + +/** + * Find an already-synced document that names the same book as this file, using + * every identity a KOSync client might have sent (see `SAME_BOOK_FILE`). + * Ordered so a byte-exact `documentHash` beats a filename-derived one, and + * skipping the dismissal sentinel, which is the user saying "not on BookHive". + */ +async function findExactSyncLink( + db: Database, + userDid: string, + contentHash: string, + uploadFilenameHash: string | null, + uploadFilenameKey: string | null, +): Promise { + const doc = await db + .selectFrom("sync_document") + .select("hiveId") + .where("userDid", "=", userDid) + .where("hiveId", "is not", null) + .where("hiveId", "!=", NO_HIVE_MATCH) + .where((eb) => + eb.or([ + eb("documentHash", "=", contentHash), + ...(uploadFilenameHash ? [eb("documentHash", "=", uploadFilenameHash)] : []), + ...(uploadFilenameKey ? [eb("filenameKey", "=", uploadFilenameKey)] : []), + ]), + ) + .orderBy(sql`CASE WHEN documentHash = ${contentHash} THEN 0 ELSE 1 END`, "asc") + .executeTakeFirst(); + return doc?.hiveId ?? null; +} + +/** KOReader stores its fraction as 0..1; anything else we treat as absent. */ +function progressPercentage(progressData: string | null | undefined): number | null { + if (!progressData) return null; + try { + const parsed = JSON.parse(progressData) as SyncProgressData; + const pct = Number(parsed.percentage); + return Number.isFinite(pct) ? pct : null; + } catch { + return null; + } +} + +export type UploadPersonalBookInput = { + db: Database; + kv: Storage; + userDid: string; + filename: string; + source: UploadSource; +}; + +export async function uploadPersonalBook( + input: UploadPersonalBookInput, +): Promise { + const { db, kv, userDid, filename, source } = input; + const quotaBytes = getStorageQuota(); + const declared = + source.kind === "stream" ? source.declaredLength : (source.bytes.length as number | undefined); + + // ── 1. Reject on the declared size before reading a byte ── + // Advisory (a chunked body has no Content-Length, and the value is + // client-asserted either way), so it is an optimisation rather than the + // control — steps 2 and 8 are what actually enforce these two limits. + if (declared !== undefined && declared > MAX_PERSONAL_BOOK_BYTES) { + return { ok: false, reason: "too-large", limitBytes: MAX_PERSONAL_BOOK_BYTES }; + } + if (declared !== undefined && declared > 0) { + const used = await getStorageUsage(db, userDid); + if (used + declared > quotaBytes) { + return { + ok: false, + reason: "quota-exceeded", + usedBytes: used, + quotaBytes, + fileBytes: declared, + }; + } + } + + // ── 2. Stream to a temp file on the same filesystem as the library ── + const tmpDir = getLibraryTmpDir(); + await ensureDir(tmpDir); + const tmp = path.join(tmpDir, `${randomUUID()}.part`); + + let size: number; + try { + size = await writeCapped(tmp, source, MAX_PERSONAL_BOOK_BYTES); + } catch (err) { + await rm(tmp, { force: true }); + if (err instanceof TooLargeError) { + return { ok: false, reason: "too-large", limitBytes: MAX_PERSONAL_BOOK_BYTES }; + } + throw err; + } + + try { + if (size === 0) return { ok: false, reason: "empty" }; + + const file = Bun.file(tmp); + + // ── 3. Format, from a 4 KB head ── + // The magic-byte check against the filename's extension is the real gate on + // what we accept; a declared Content-Type is client-asserted and worthless. + const head = new Uint8Array(await file.slice(0, FORMAT_HEAD_BYTES).arrayBuffer()); + const formatInfo: FormatInfo = detectFormat(head, filename); + if (formatInfo.format === "unknown") { + return { ok: false, reason: "unsupported-format", filename }; + } + + // ── 4. Content hash, from twelve 1 KB windows ── + const contentHash = await koreaderPartialMD5File(file, size); + + // ── 5. Duplicate check, BEFORE the parse ── + // Deliberately ahead of both the parse and the disk commit so a re-upload + // costs neither. `src/routes/library.test.ts` leans on this ordering to + // exercise the duplicate path without touching the library directory. + const duplicate = await db + .selectFrom("personal_book") + .select("id") + .where("userDid", "=", userDid) + .where("contentHash", "=", contentHash) + .executeTakeFirst(); + if (duplicate) return { ok: false, reason: "duplicate", contentHash }; + + // ── 6. Parse — the only full-buffer step ── + let metadata: BookMetadata; + try { + metadata = await parseSemaphore.run(async () => { + const bytes = new Uint8Array(await file.arrayBuffer()); + return parseBook(bytes, filename, formatInfo); + }); + } catch (err) { + if (err instanceof SemaphoreFullError || err instanceof SemaphoreTimeoutError) { + return { ok: false, reason: "busy" }; + } + throw err; + } + + // ── 7. Cover, gated ── + // `coverPath IS NOT NULL` is the only signal driving `coverUrl` on the web + // library, the OPDS feed and the XRPC book view, so storing an unvalidated + // cover produces a dead URL and a blank box in all three. + const cover = + metadata.cover && (await isUsableCover(metadata.cover.bytes)) ? metadata.cover : undefined; + + // ── 8. Link, then insert under quota ── + const uploadFilenameHash = koreaderFilenameHash(filename); + const uploadFilenameKey = filenameKey(filename); + + // Exact first, fuzzy only on a miss. The XRPC path used to run the fuzzy + // matcher first, which let a title/author guess beat a byte-exact + // documentHash match — strictly wrong, and it paid for up to four FTS + // queries on the common path where the exact lookup would have answered. + let hiveId = await findExactSyncLink( + db, + userDid, + contentHash, + uploadFilenameHash, + uploadFilenameKey, + ); + if (!hiveId) { + hiveId = await matchSyncDocument(db, { + title: metadata.title, + authors: metadata.authors, + filename, + }); + } + + const now = new Date().toISOString(); + const filePath = bookFilePath(userDid, contentHash, formatInfo.ext); + const coverPath = cover ? coverFilePath(userDid, contentHash, cover.ext) : null; + + const inserted = await insertIfUnderQuota( + db, + { + userDid, + contentHash, + hiveId, + filename, + filenameHash: uploadFilenameHash, + filenameKey: uploadFilenameKey, + title: metadata.title, + // `parseBook` returns "" on every fallback. Normalise to NULL so + // `WHERE authors IS NULL` means what it looks like it means — the two + // are identical to JS truthiness and completely different to SQL. + authors: metadata.authors || null, + language: metadata.language || null, + format: formatInfo.format, + mime: formatInfo.mime, + filePath, + coverPath, + coverMime: cover?.mime ?? null, + sizeBytes: size, + createdAt: now, + updatedAt: now, + }, + quotaBytes, + ); + if (!inserted) { + const used = await getStorageUsage(db, userDid); + return { ok: false, reason: "quota-exceeded", usedBytes: used, quotaBytes, fileBytes: size }; + } + + // ── 9. Commit the bytes: rename, not copy ── + await ensureDir(personalBookDir(userDid, contentHash)); + await rename(tmp, filePath); + if (cover && coverPath) await Bun.write(coverPath, cover.bytes); + + // ── 10. Propagate the link outward ── + if (hiveId) { + // Any document the device has been pushing progress for that never + // matched now points at this book — so the next push bridges instead of + // being dropped. + const linked = await db + .selectFrom("sync_document") + .select(["id", "progressData"]) + .where("userDid", "=", userDid) + .where("hiveId", "is", null) + .where((eb) => + eb.or([ + eb("documentHash", "=", contentHash), + ...(uploadFilenameHash ? [eb("documentHash", "=", uploadFilenameHash)] : []), + ...(uploadFilenameKey ? [eb("filenameKey", "=", uploadFilenameKey)] : []), + ]), + ) + .execute(); + + if (linked.length > 0) { + await db + .updateTable("sync_document") + .set({ hiveId }) + .where( + "id", + "in", + linked.map((d) => d.id), + ) + .execute(); + + // The percentage those documents already recorded has been sitting + // unused; without this it stays that way until the device next syncs. + for (const doc of linked) { + const pct = progressPercentage(doc.progressData); + if (pct !== null) await bridgeProgressToUserBook(db, kv, userDid, hiveId, pct); + } + } + + await db + .updateTable("user_book") + .set({ owned: 1 }) + .where("userDid", "=", userDid) + .where("hiveId", "=", hiveId) + .where("owned", "=", 0) + .execute(); + } + + return { + ok: true, + book: { + contentHash, + title: metadata.title, + authors: metadata.authors || undefined, + language: metadata.language || undefined, + format: formatInfo.format, + mime: formatInfo.mime, + sizeBytes: size, + createdAt: now, + updatedAt: now, + hiveId: hiveId ?? undefined, + coverUrl: coverPath ? `/library/covers/${contentHash}` : undefined, + }, + storageUsedBytes: await getStorageUsage(db, userDid), + storageQuotaBytes: quotaBytes, + }; + } finally { + // No-op once the rename has happened; the safety net for every path that + // returns before it. + await rm(tmp, { force: true }); + } +} + +/** + * Delete `.part` files left behind by a process that died between the write and + * the rename. Runs on the primary worker at startup; an hour is well past any + * live upload (the parse semaphore times out at 30s). + */ +export async function sweepStaleUploads(maxAgeMs = 60 * 60 * 1000): Promise { + const tmpDir = getLibraryTmpDir(); + let names: string[]; + try { + names = await readdir(tmpDir); + } catch { + return 0; // never uploaded anything on this host + } + const cutoff = Date.now() - maxAgeMs; + let removed = 0; + for (const name of names) { + if (!name.endsWith(".part")) continue; + const full = path.join(tmpDir, name); + try { + const info = await stat(full); + if (info.mtimeMs < cutoff) { + await rm(full, { force: true }); + removed++; + } + } catch { + /* raced with another sweep or the upload itself */ + } + } + return removed; +} diff --git a/src/xrpc/auth.test.ts b/src/xrpc/auth.test.ts new file mode 100644 index 00000000..701123a6 --- /dev/null +++ b/src/xrpc/auth.test.ts @@ -0,0 +1,330 @@ +/** + * Service auth on /xrpc/*. + * + * The verifier is NOT stubbed — a stubbed one would test nothing, and the + * negative cases (wrong audience, wrong lxm, expired, tampered signature) are + * the entire point. Instead a real `ServiceJwtVerifier` runs against a real + * `createServiceJwt`-signed token, with a static one-method `DidDocumentResolver` + * standing in for the network. That exercises every check the production path + * runs, deterministically and offline. + */ + +import { describe, it, expect, beforeEach, afterEach } from "bun:test"; +import { Database as DatabaseSync } from "bun:sqlite"; +import { Hono } from "hono"; +import { Kysely, SqliteDialect } from "kysely"; +import { createStorage } from "unstorage"; +import memoryDriver from "unstorage/drivers/memory"; +import { rm } from "node:fs/promises"; +import path from "node:path"; +import type { Storage } from "unstorage"; +import { P256PrivateKeyExportable } from "@atcute/crypto"; +import { ServiceJwtVerifier, createServiceJwt } from "@atcute/xrpc-server/auth"; +import type { Did, Nsid } from "@atcute/lexicons"; + +import { wrapBunSqliteForKysely } from "../bun-sqlite-kysely"; +import type { AppContext, AppEnv } from "../context"; +import { migrateToLatest, type DatabaseSchema, type Database } from "../db"; +import { makeEpub } from "../utils/bookMetadata/testFixtures"; +import { markAccount, isKnownAccount } from "../utils/account"; +import { personalBookDir } from "../utils/personalLibrary"; +import { createXrpcRouter, type XrpcContext } from "./router"; + +const DID = "did:plc:testuser" as Did; +const STRANGER = "did:plc:neverheardofthem" as Did; +const SERVICE_DID = "did:plc:enu2j5xjlqsjaylv3du4myh4" as Did; + +let db: Database; +let kv: Storage; +let keypair: P256PrivateKeyExportable; +let strangerKeypair: P256PrivateKeyExportable; + +async function createTestDb(): Promise { + const sqlite = new DatabaseSync(":memory:"); + sqlite.exec("PRAGMA journal_mode = WAL"); + const database = new Kysely({ + dialect: new SqliteDialect({ database: wrapBunSqliteForKysely(sqlite) }), + }); + await migrateToLatest(database, sqlite); + return database; +} + +/** A DID document carrying the given key as its `#atproto` verification method. */ +async function didDocFor(did: Did, key: P256PrivateKeyExportable) { + return { + id: did, + verificationMethod: [ + { + id: `${did}#atproto`, + type: "Multikey", + controller: did, + publicKeyMultibase: await key.exportPublicKey("multikey"), + }, + ], + }; +} + +/** Resolves both test identities; the network is never touched. */ +async function staticResolver() { + const docs = new Map([ + [DID, await didDocFor(DID, keypair)], + [STRANGER, await didDocFor(STRANGER, strangerKeypair)], + ]); + return { + resolve: async (did: string) => { + const doc = docs.get(did); + if (!doc) throw new Error(`no did doc for ${did}`); + return doc; + }, + }; +} + +async function createApp(opts: { audiences?: string[]; maxAge?: number; enabled?: boolean } = {}) { + const resolver = await staticResolver(); + const verifier = + opts.enabled === false + ? null + : new ServiceJwtVerifier({ + acceptAudiences: (opts.audiences ?? [SERVICE_DID]) as Did[], + resolver: resolver as never, + maxAge: opts.maxAge ?? 3600, + }); + + const app = new Hono(); + app.use("*", async (c, next) => { + c.set("ctx", { + db, + kv, + resolver: { resolveDidsToHandles: async () => ({}) }, + // No cookie session: these tests are exclusively about the Bearer path. + getSessionAgent: async () => null, + baseIdResolver: { handle: { resolve: async () => undefined } }, + addWideEventContext: () => {}, + serviceJwtVerifier: verifier, + isKnownAccount: (did: string) => isKnownAccount({ db, kv }, did), + } as unknown as AppContext); + await next(); + }); + createXrpcRouter( + app as never, + { + searchBooks: async () => [], + ensureBookIdentifiersCurrent: async () => {}, + getProfile: async () => null, + } as never, + ); + return app; +} + +function token( + opts: { + lxm?: string; + audience?: string; + issuer?: Did; + key?: P256PrivateKeyExportable; + expiresIn?: number; + issuedAt?: number; + } = {}, +) { + return createServiceJwt({ + keypair: opts.key ?? keypair, + issuer: opts.issuer ?? DID, + audience: (opts.audience ?? SERVICE_DID) as Did, + lxm: (opts.lxm ?? "buzz.bookhive.listPersonalShelves") as Nsid, + ...(opts.expiresIn !== undefined ? { expiresIn: opts.expiresIn } : {}), + ...(opts.issuedAt !== undefined ? { issuedAt: opts.issuedAt } : {}), + }); +} + +beforeEach(async () => { + db = await createTestDb(); + kv = createStorage({ driver: memoryDriver() }); + keypair = await P256PrivateKeyExportable.createKeypair(); + strangerKeypair = await P256PrivateKeyExportable.createKeypair(); + // DID has used BookHive before; STRANGER has not. + await markAccount(kv, DID); +}); + +afterEach(async () => { + await rm(path.dirname(personalBookDir(DID, "x")), { recursive: true, force: true }).catch( + () => {}, + ); +}); + +describe("service auth — the happy path", () => { + it("authenticates a query with a Bearer service token", async () => { + const app = await createApp(); + const res = await app.request("/xrpc/buzz.bookhive.listPersonalShelves", { + headers: { authorization: `Bearer ${await token()}` }, + }); + expect(res.status).toBe(200); + expect((await res.json()) as { totalBooks: number }).toHaveProperty("totalBooks", 0); + }); + + it("authenticates an upload, the whole point of the feature", async () => { + const app = await createApp(); + const bytes = makeEpub({ title: "Dune" }); + const res = await app.request("/xrpc/buzz.bookhive.uploadPersonalBook?filename=Dune.epub", { + method: "POST", + body: bytes as BodyInit, + headers: { + authorization: `Bearer ${await token({ lxm: "buzz.bookhive.uploadPersonalBook" })}`, + "content-type": "application/epub+zip", + "content-length": String(bytes.length), + }, + }); + expect(res.status).toBe(200); + + // And the row landed under the token's issuer, not some other DID. + const row = await db + .selectFrom("personal_book") + .select(["userDid", "title"]) + .executeTakeFirstOrThrow(); + expect(row.userDid).toBe(DID); + expect(row.title).toBe("Dune"); + }); + + it("accepts the #fragment audience form as well as the bare DID", async () => { + // atcute compares audiences by exact string, so both spellings must be + // listed. This is what will keep clients working when the DID document + // gains a #bookhive_appview service entry. + const app = await createApp({ audiences: [SERVICE_DID, `${SERVICE_DID}#bookhive_appview`] }); + const res = await app.request("/xrpc/buzz.bookhive.listPersonalShelves", { + headers: { + authorization: `Bearer ${await token({ audience: `${SERVICE_DID}#bookhive_appview` })}`, + }, + }); + expect(res.status).toBe(200); + }); +}); + +describe("service auth — rejections", () => { + const cases: { name: string; make: () => Promise; lxm?: string }[] = [ + { + name: "a token minted for a different audience", + make: () => token({ audience: "did:web:someone-else.example" }), + }, + { + name: "a token bound to a different method", + make: () => token({ lxm: "buzz.bookhive.getPersonalLibrary" }), + }, + { + name: "an expired token", + make: () => token({ issuedAt: Math.floor(Date.now() / 1000) - 600, expiresIn: 60 }), + }, + { + name: "a token signed by the wrong key", + // Issued as DID, but signed with the stranger's key — the DID document + // for DID carries a different public key, so the signature can't verify. + make: () => token({ key: strangerKeypair }), + }, + ]; + + for (const c of cases) { + it(`401s ${c.name}`, async () => { + const app = await createApp(); + const res = await app.request("/xrpc/buzz.bookhive.listPersonalShelves", { + headers: { authorization: `Bearer ${await c.make()}` }, + }); + expect(res.status).toBe(401); + }); + } + + it("401s a token whose signature does not cover its payload", async () => { + // Splice a *different* token's signature onto this one's header+payload. + // Deliberately not "flip the last base64url character": that char carries + // padding bits, so flipping it can decode to the identical signature bytes + // and the token still verifies — which made this test flaky. + const app = await createApp(); + const [header, payload] = (await token()).split("."); + const [, , otherSig] = (await token({ key: strangerKeypair })).split("."); + const forged = `${header}.${payload}.${otherSig}`; + + const res = await app.request("/xrpc/buzz.bookhive.listPersonalShelves", { + headers: { authorization: `Bearer ${forged}` }, + }); + expect(res.status).toBe(401); + }); + + it("401s a token older than the configured max-age window", async () => { + const app = await createApp({ maxAge: 300 }); + const res = await app.request("/xrpc/buzz.bookhive.listPersonalShelves", { + headers: { authorization: `Bearer ${await token({ expiresIn: 3600 })}` }, + }); + expect(res.status).toBe(401); + }); + + it("401s a valid token from a DID that has never used BookHive", async () => { + // The gate that stops any identity on the network opening a storage quota + // on our disk. The token itself is perfectly valid. + const app = await createApp(); + const res = await app.request("/xrpc/buzz.bookhive.listPersonalShelves", { + headers: { + authorization: `Bearer ${await token({ issuer: STRANGER, key: strangerKeypair })}`, + }, + }); + expect(res.status).toBe(401); + expect(((await res.json()) as { message: string }).message).toContain("No BookHive account"); + }); + + it("401s garbage in the Authorization header", async () => { + const app = await createApp(); + const res = await app.request("/xrpc/buzz.bookhive.listPersonalShelves", { + headers: { authorization: "Bearer not-a-jwt" }, + }); + expect(res.status).toBe(401); + }); + + it("401s when service auth is disabled", async () => { + const app = await createApp({ enabled: false }); + const res = await app.request("/xrpc/buzz.bookhive.listPersonalShelves", { + headers: { authorization: `Bearer ${await token()}` }, + }); + expect(res.status).toBe(401); + expect(((await res.json()) as { message: string }).message).toContain("not enabled"); + }); +}); + +describe("service auth — what it deliberately cannot do", () => { + it("refuses a method that writes to the user's repo", async () => { + // `createList` puts a record in the caller's repository, which needs an + // OAuth grant. A service token proves key control, not that we hold one. + const app = await createApp(); + const res = await app.request("/xrpc/buzz.bookhive.createList", { + method: "POST", + body: JSON.stringify({ name: "Sci-Fi" }), + headers: { + authorization: `Bearer ${await token({ lxm: "buzz.bookhive.createList" })}`, + "content-type": "application/json", + }, + }); + expect(res.status).toBe(401); + const message = ((await res.json()) as { message: string }).message; + expect(message).toContain("OAuth session"); + }); +}); + +describe("isKnownAccount", () => { + it("backfills from an existing account's durable traces", async () => { + // Every current user predates the marker, so the probe is what makes + // service auth usable for them on day one. + const freshKv = createStorage({ driver: memoryDriver() }); + expect(await isKnownAccount({ db, kv: freshKv }, DID)).toBe(false); + + await db + .insertInto("sync_document") + .values({ + userDid: DID, + provider: "kosync", + documentHash: "abc", + progressData: "{}", + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + }) + .execute(); + + expect(await isKnownAccount({ db, kv: freshKv }, DID)).toBe(true); + // ...and memoised, so the four-way probe runs once per DID. + expect(await freshKv.hasItem(`account:${DID}`)).toBe(true); + }); +}); diff --git a/src/xrpc/auth.ts b/src/xrpc/auth.ts new file mode 100644 index 00000000..ed0adb71 --- /dev/null +++ b/src/xrpc/auth.ts @@ -0,0 +1,92 @@ +/** + * Authentication for `/xrpc/*`. + * + * Two credentials are accepted: + * + * - The `sid` iron-session cookie, which is what the web app and the iOS app + * have always used. + * - An **atproto inter-service auth JWT** as `Authorization: Bearer ` — + * https://atproto.com/specs/xrpc#inter-service-authentication-jwt. The client + * asks its own PDS for a token via `com.atproto.server.getServiceAuth`, + * bound to an audience (us) and an `lxm` (the one method it wants to call); + * the PDS signs it with the account's repo signing key, and we verify it by + * resolving the issuer's DID document. This is the canonical mechanism for a + * third-party service exposing its own XRPC methods, and it is what makes the + * personal library reachable from a script or an e-reader rather than only + * from a browser session. + * + * The one thing service auth cannot do is write to the user's repo: it proves + * control of a signing key, not that we hold an OAuth grant for that account. + * `AuthMode` is how a method declares which it needs. + */ + +import { AuthRequiredError } from "@atcute/xrpc-server"; +import type { ServiceJwtVerifier } from "@atcute/xrpc-server/auth"; +import type { Nsid } from "@atcute/lexicons"; +import type { SessionClient } from "../auth/client"; + +/** + * What a method requires of its caller. + * + * `identity` — we only need to know *who* they are. Every personal-library and + * sync method is in this class: none of them touch the session agent for + * anything but `.did` (progress bridging writes `user_book` and queues a + * deferred PDS write via `sync_pending:`, rather than writing inline). + * `pdsWrite` — the handler puts a record in the user's repository, which needs + * a live OAuth session. Only the book-list procedures are in this class. + */ +export type AuthMode = "identity" | "pdsWrite"; + +export type XrpcAuth = + | { did: string; method: "session"; agent: SessionClient } + | { did: string; method: "service"; agent: null }; + +export type XrpcAuthContext = { + getSessionAgent: () => Promise; + serviceJwtVerifier?: ServiceJwtVerifier | null; + isKnownAccount?: (did: string) => Promise; +}; + +export async function resolveXrpcAuth( + ctx: XrpcAuthContext, + request: Request, + opts: { lxm: Nsid; mode: AuthMode }, +): Promise { + const authorization = request.headers.get("authorization"); + + // Bearer wins when both are somehow present: a browser never sends one and a + // programmatic client never has our cookie, so a request carrying both is + // stating its intent. + if (authorization !== null && /^bearer\s/i.test(authorization)) { + if (!ctx.serviceJwtVerifier) { + throw new AuthRequiredError({ message: "Service auth is not enabled on this server" }); + } + if (opts.mode === "pdsWrite") { + throw new AuthRequiredError({ + message: + `${opts.lxm} writes a record to your repository, which needs an OAuth session; ` + + `service auth cannot provide one. Sign in at bookhive.buzz to use this method.`, + }); + } + + // Throws AuthRequiredError (401, with a WWW-Authenticate: Bearer challenge) + // on every failure path: missing or malformed token, bad signature, wrong + // audience, wrong lxm, expired, outside the max-age window, or replayed. + const { issuer } = await ctx.serviceJwtVerifier.verifyRequest(request, { lxm: opts.lxm }); + + // A valid token proves control of an atproto identity, not that the + // identity has ever used BookHive. Without this gate any DID on the network + // could open a storage quota's worth of space on our disk. + if (ctx.isKnownAccount && !(await ctx.isKnownAccount(issuer))) { + throw new AuthRequiredError({ + message: "No BookHive account for this DID — sign in at bookhive.buzz once first", + }); + } + + return { did: issuer, method: "service", agent: null }; + } + + const agent = await ctx.getSessionAgent(); + if (!agent) throw new AuthRequiredError({ message: "Authentication required" }); + return { did: agent.did, method: "session", agent }; +} diff --git a/src/xrpc/personalLibrary.test.ts b/src/xrpc/personalLibrary.test.ts new file mode 100644 index 00000000..020c34bf --- /dev/null +++ b/src/xrpc/personalLibrary.test.ts @@ -0,0 +1,505 @@ +/** + * The first tests of `src/xrpc/router.ts`. + * + * Scoped to the personal-library methods rather than named `router.test.ts`: + * that file is 2000+ lines and 40 methods, and one suite per region stays + * reviewable (and signposts `src/xrpc/lists.test.ts` for whoever needs it next). + */ + +import { describe, it, expect, beforeEach, afterEach } from "bun:test"; +import { Database as DatabaseSync } from "bun:sqlite"; +import { Hono } from "hono"; +import { Kysely, SqliteDialect } from "kysely"; +import { createStorage } from "unstorage"; +import memoryDriver from "unstorage/drivers/memory"; +import { readdir, rm } from "node:fs/promises"; +import path from "node:path"; +import type { Storage } from "unstorage"; + +import { wrapBunSqliteForKysely } from "../bun-sqlite-kysely"; +import type { AppContext, AppEnv } from "../context"; +import { migrateToLatest, type DatabaseSchema, type Database } from "../db"; +import { koreaderPartialMD5 } from "../utils/bookMetadata/index"; +import { makeEpub, makeFb2 } from "../utils/bookMetadata/testFixtures"; +import { + bookFilePath, + getLibraryTmpDir, + getStorageQuota, + personalBookDir, +} from "../utils/personalLibrary"; +import { createXrpcRouter, type XrpcContext } from "./router"; + +const DID = "did:plc:testuser"; +const OTHER_DID = "did:plc:someoneelse"; + +type TestApp = Hono; + +let db: Database; +let kv: Storage; +let wideEvent: Record; + +async function createTestDb(): Promise { + const sqlite = new DatabaseSync(":memory:"); + sqlite.exec("PRAGMA journal_mode = WAL"); + const database = new Kysely({ + dialect: new SqliteDialect({ database: wrapBunSqliteForKysely(sqlite) }), + }); + await migrateToLatest(database, sqlite); + return database; +} + +function createApp(did: string | null = DID): TestApp { + const app = new Hono(); + app.use("*", async (c, next) => { + c.set("ctx", { + db, + kv, + resolver: { resolveDidsToHandles: async () => ({}) }, + getSessionAgent: async () => (did ? { did } : null), + baseIdResolver: { handle: { resolve: async () => undefined } }, + addWideEventContext: (fields: Record) => Object.assign(wideEvent, fields), + } as unknown as AppContext); + await next(); + }); + createXrpcRouter( + app as never, + { + searchBooks: async () => [], + ensureBookIdentifiersCurrent: async () => {}, + getProfile: async () => null, + } as never, + ); + return app; +} + +/** POST an ebook as a raw body, the way a programmatic client would. */ +function uploadRequest( + app: TestApp, + bytes: Uint8Array, + filename: string, + init: { contentType?: string; contentLength?: boolean } = {}, +) { + const headers: Record = { + "content-type": init.contentType ?? "application/epub+zip", + }; + if (init.contentLength !== false) headers["content-length"] = String(bytes.length); + return app.request( + `/xrpc/buzz.bookhive.uploadPersonalBook?filename=${encodeURIComponent(filename)}`, + { method: "POST", body: bytes as BodyInit, headers }, + ); +} + +async function tmpEntries(): Promise { + try { + return (await readdir(getLibraryTmpDir())).filter((n) => n.endsWith(".part")); + } catch { + return []; + } +} + +beforeEach(async () => { + db = await createTestDb(); + kv = createStorage({ driver: memoryDriver() }); + wideEvent = {}; +}); + +afterEach(async () => { + for (const did of [DID, OTHER_DID]) { + await rm(path.dirname(personalBookDir(did, "x")), { recursive: true, force: true }).catch( + () => {}, + ); + } + for (const name of await tmpEntries()) { + await rm(path.join(getLibraryTmpDir(), name), { force: true }); + } +}); + +describe("XRPC uploadPersonalBook", () => { + it("accepts a raw ebook body and stores it", async () => { + const app = createApp(); + const bytes = makeEpub({ title: "Dune", authors: ["Frank Herbert"] }); + + const res = await uploadRequest(app, bytes, "Dune.epub"); + expect(res.status).toBe(200); + + const body = (await res.json()) as { + book: { contentHash: string; title: string; authors?: string; sizeBytes: number }; + storageUsedBytes: number; + storageQuotaBytes: number; + }; + expect(body.book.title).toBe("Dune"); + expect(body.book.authors).toBe("Frank Herbert"); + expect(body.book.sizeBytes).toBe(bytes.length); + expect(body.book.contentHash).toBe(koreaderPartialMD5(bytes)); + expect(body.storageUsedBytes).toBe(bytes.length); + expect(body.storageQuotaBytes).toBe(getStorageQuota()); + + expect(await Bun.file(bookFilePath(DID, body.book.contentHash, "epub")).exists()).toBe(true); + expect(await tmpEntries()).toEqual([]); + }); + + it("accepts application/octet-stream, which is what real clients send", async () => { + // Mobile document pickers and `curl --data-binary` both report this; the + // lexicon's MIME list documents intent, but detectFormat is the real gate. + const res = await uploadRequest(createApp(), makeEpub(), "x.epub", { + contentType: "application/octet-stream", + }); + expect(res.status).toBe(200); + }); + + it("rejects a content type outside the lexicon's list before the handler runs", async () => { + const res = await uploadRequest(createApp(), makeEpub(), "x.epub", { + contentType: "application/json", + }); + expect(res.status).toBe(400); + expect(((await res.json()) as { error: string }).error).toBe("InvalidRequest"); + }); + + it("requires the filename parameter", async () => { + const app = createApp(); + const res = await app.request("/xrpc/buzz.bookhive.uploadPersonalBook", { + method: "POST", + body: makeEpub() as BodyInit, + headers: { "content-type": "application/epub+zip" }, + }); + expect(res.status).toBe(400); + }); + + it("uses the filename to tell zip containers apart", async () => { + // An EPUB and a CBZ are both zip archives; only the extension distinguishes + // them, which is why `filename` is required rather than a header. + const app = createApp(); + const res = await uploadRequest(app, makeEpub(), "book.cbz", { + contentType: "application/vnd.comicbook+zip", + }); + expect(res.status).toBe(200); + expect(((await res.json()) as { book: { format: string } }).book.format).toBe("cbz"); + }); + + it("401s without a session", async () => { + const res = await uploadRequest(createApp(null), makeEpub(), "x.epub"); + expect(res.status).toBe(401); + }); + + it("409s a duplicate", async () => { + const app = createApp(); + const bytes = makeEpub(); + expect((await uploadRequest(app, bytes, "x.epub")).status).toBe(200); + + const res = await uploadRequest(app, bytes, "x.epub"); + expect(res.status).toBe(409); + expect(((await res.json()) as { error: string }).error).toBe("AlreadyExists"); + }); + + it("400s an unsupported format", async () => { + const res = await uploadRequest( + createApp(), + new TextEncoder().encode("plain text, not a book"), + "notes.epub", + ); + expect(res.status).toBe(400); + expect(((await res.json()) as { error: string }).error).toBe("InvalidRequest"); + }); + + it("413s when the upload would cross the storage quota", async () => { + const bytes = makeEpub(); + await db + .insertInto("personal_book") + .values({ + userDid: DID, + contentHash: "seeded", + filename: "seeded.epub", + title: "Seeded", + format: "epub", + mime: "application/epub+zip", + filePath: "/tmp/seeded.epub", + sizeBytes: getStorageQuota() - bytes.length + 1, + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + }) + .execute(); + + const res = await uploadRequest(createApp(), bytes, "x.epub"); + expect(res.status).toBe(413); + const body = (await res.json()) as { error: string }; + expect(body.error).toBe("QuotaExceeded"); + expect(await tmpEntries()).toEqual([]); + }); + + it("records a deliberate 4xx on the wide event without a stack", async () => { + // The registration wrapper decides this from `status < 500`, and nothing + // covered it before. + await uploadRequest(createApp(), new TextEncoder().encode("nope"), "x.epub"); + expect(wideEvent["xrpc_handler"]).toBe("threw"); + const error = wideEvent["error"] as { message?: string; stack?: string } | undefined; + expect(error?.message).toBeTruthy(); + expect(error?.stack).toBeUndefined(); + }); + + it("keeps two users' identical uploads separate", async () => { + const bytes = makeFb2(); + expect( + ( + await uploadRequest(createApp(DID), bytes, "x.fb2", { + contentType: "application/x-fictionbook+xml", + }) + ).status, + ).toBe(200); + expect( + ( + await uploadRequest(createApp(OTHER_DID), bytes, "x.fb2", { + contentType: "application/x-fictionbook+xml", + }) + ).status, + ).toBe(200); + + const rows = await db.selectFrom("personal_book").select(["userDid"]).execute(); + expect(rows.map((r) => r.userDid).sort()).toEqual([OTHER_DID, DID].sort()); + }); + + it("streams a body with no content-length", async () => { + const res = await uploadRequest(createApp(), makeEpub(), "x.epub", { contentLength: false }); + expect(res.status).toBe(200); + }); +}); + +describe("XRPC getPersonalBookFile", () => { + it("serves the stored bytes with download headers", async () => { + const app = createApp(); + const bytes = makeEpub(); + const hash = ( + (await (await uploadRequest(app, bytes, "Dune.epub")).json()) as { + book: { contentHash: string }; + } + ).book.contentHash; + + const res = await app.request(`/xrpc/buzz.bookhive.getPersonalBookFile?contentHash=${hash}`); + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toBe("application/epub+zip"); + expect(res.headers.get("content-length")).toBe(String(bytes.length)); + expect(res.headers.get("content-disposition")).toContain("Dune.epub"); + expect(res.headers.get("etag")).toBe(`"${hash}"`); + expect(Array.from(new Uint8Array(await res.arrayBuffer()))).toEqual(Array.from(bytes)); + }); + + it("answers If-None-Match with a 304 and no body", async () => { + // The reason this matters: without it an e-reader re-downloads every book + // on every scheduled sync. + const app = createApp(); + const hash = ( + (await (await uploadRequest(app, makeEpub(), "x.epub")).json()) as { + book: { contentHash: string }; + } + ).book.contentHash; + + const res = await app.request(`/xrpc/buzz.bookhive.getPersonalBookFile?contentHash=${hash}`, { + headers: { "if-none-match": `"${hash}"` }, + }); + expect(res.status).toBe(304); + expect(await res.text()).toBe(""); + }); + + it("404s another user's book rather than 403", async () => { + const hash = ( + (await (await uploadRequest(createApp(DID), makeEpub(), "x.epub")).json()) as { + book: { contentHash: string }; + } + ).book.contentHash; + + const res = await createApp(OTHER_DID).request( + `/xrpc/buzz.bookhive.getPersonalBookFile?contentHash=${hash}`, + ); + // 404, not 403: a different status would confirm the book exists. + expect(res.status).toBe(404); + }); + + it("401s without a session", async () => { + const res = await createApp(null).request( + "/xrpc/buzz.bookhive.getPersonalBookFile?contentHash=whatever", + ); + expect(res.status).toBe(401); + }); +}); + +describe("XRPC getPersonalBookCover", () => { + it("serves the extracted cover", async () => { + const app = createApp(); + const hash = ( + (await (await uploadRequest(app, makeEpub(), "x.epub")).json()) as { + book: { contentHash: string }; + } + ).book.contentHash; + + const res = await app.request(`/xrpc/buzz.bookhive.getPersonalBookCover?contentHash=${hash}`); + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toBe("image/png"); + expect(res.headers.get("etag")).toBe(`"${hash}-cover"`); + }); + + it("answers If-None-Match on the cover", async () => { + const app = createApp(); + const hash = ( + (await (await uploadRequest(app, makeEpub(), "x.epub")).json()) as { + book: { contentHash: string }; + } + ).book.contentHash; + + const res = await app.request(`/xrpc/buzz.bookhive.getPersonalBookCover?contentHash=${hash}`, { + headers: { "if-none-match": `"${hash}-cover"` }, + }); + expect(res.status).toBe(304); + }); + + it("404s a book with neither a stored cover nor a catalog entry", async () => { + const app = createApp(); + // FB2 fixture carries no cover image. + const hash = ( + (await ( + await uploadRequest(app, makeFb2(), "x.fb2", { + contentType: "application/x-fictionbook+xml", + }) + ).json()) as { book: { contentHash: string } } + ).book.contentHash; + + const res = await app.request(`/xrpc/buzz.bookhive.getPersonalBookCover?contentHash=${hash}`); + expect(res.status).toBe(404); + }); +}); + +describe("XRPC getPersonalLibrary — search, sort and storage", () => { + async function seedLibrary(app: TestApp) { + await uploadRequest(app, makeEpub({ title: "Dune", authors: ["Frank Herbert"] }), "Dune.epub"); + await uploadRequest( + app, + makeEpub({ title: "Neuromancer", authors: ["William Gibson"] }), + "Neuromancer.epub", + ); + await uploadRequest( + app, + makeEpub({ title: "Ancillary Justice", authors: ["Ann Leckie"] }), + "Ancillary.epub", + ); + } + + it("filters on title or author, matching the OPDS search feed", async () => { + const app = createApp(); + await seedLibrary(app); + + const byTitle = (await ( + await app.request("/xrpc/buzz.bookhive.getPersonalLibrary?q=neuro") + ).json()) as { books: { title: string }[]; total: number }; + expect(byTitle.books.map((b) => b.title)).toEqual(["Neuromancer"]); + expect(byTitle.total).toBe(1); + + const byAuthor = (await ( + await app.request("/xrpc/buzz.bookhive.getPersonalLibrary?q=Leckie") + ).json()) as { books: { title: string }[] }; + expect(byAuthor.books.map((b) => b.title)).toEqual(["Ancillary Justice"]); + }); + + it("sorts by title and by author on request", async () => { + const app = createApp(); + await seedLibrary(app); + + const byTitle = (await ( + await app.request("/xrpc/buzz.bookhive.getPersonalLibrary?sort=title") + ).json()) as { books: { title: string }[] }; + expect(byTitle.books.map((b) => b.title)).toEqual(["Ancillary Justice", "Dune", "Neuromancer"]); + + const byAuthor = (await ( + await app.request("/xrpc/buzz.bookhive.getPersonalLibrary?sort=author") + ).json()) as { books: { authors?: string }[] }; + expect(byAuthor.books.map((b) => b.authors)).toEqual([ + "Ann Leckie", + "Frank Herbert", + "William Gibson", + ]); + }); + + it("defaults to newest first, and does not switch when q is set", async () => { + const app = createApp(); + await seedLibrary(app); + const res = (await (await app.request("/xrpc/buzz.bookhive.getPersonalLibrary")).json()) as { + books: { title: string }[]; + }; + expect(res.books[0]!.title).toBe("Ancillary Justice"); + }); + + it("reports storage usage and the extra view fields", async () => { + const app = createApp(); + const bytes = makeEpub({ title: "Dune" }); + await uploadRequest(app, bytes, "Dune.epub"); + + const res = (await (await app.request("/xrpc/buzz.bookhive.getPersonalLibrary")).json()) as { + books: { filename: string; hasLocalCover: boolean }[]; + storage: { usedBytes: number; quotaBytes: number }; + }; + expect(res.storage).toEqual({ usedBytes: bytes.length, quotaBytes: getStorageQuota() }); + expect(res.books[0]!.filename).toBe("Dune.epub"); + expect(res.books[0]!.hasLocalCover).toBe(true); + }); +}); + +describe("XRPC listPersonalShelves", () => { + it("returns shelves with counts plus library totals", async () => { + const app = createApp(); + await uploadRequest(app, makeEpub({ title: "Dune" }), "Dune.epub"); + const bookRow = await db.selectFrom("personal_book").select("id").executeTakeFirstOrThrow(); + + const shelf = await db + .insertInto("personal_shelf") + .values({ + userDid: DID, + name: "Sci-Fi", + description: "space", + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + }) + .returning("id") + .executeTakeFirstOrThrow(); + await db + .insertInto("personal_shelf_item") + .values({ + shelfId: shelf.id, + personalBookId: bookRow.id, + createdAt: "2026-08-01T00:00:00.000Z", + }) + .execute(); + + const res = (await (await app.request("/xrpc/buzz.bookhive.listPersonalShelves")).json()) as { + shelves: { id: number; name: string; description?: string; bookCount: number }[]; + totalBooks: number; + storage: { usedBytes: number; quotaBytes: number }; + }; + + expect(res.shelves).toHaveLength(1); + expect(res.shelves[0]!.name).toBe("Sci-Fi"); + expect(res.shelves[0]!.description).toBe("space"); + expect(res.shelves[0]!.bookCount).toBe(1); + expect(res.totalBooks).toBe(1); + expect(res.storage.quotaBytes).toBe(getStorageQuota()); + }); + + it("does not show another user's shelves", async () => { + await db + .insertInto("personal_shelf") + .values({ + userDid: OTHER_DID, + name: "Theirs", + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + }) + .execute(); + + const res = (await ( + await createApp(DID).request("/xrpc/buzz.bookhive.listPersonalShelves") + ).json()) as { shelves: unknown[]; totalBooks: number }; + expect(res.shelves).toHaveLength(0); + expect(res.totalBooks).toBe(0); + }); + + it("401s without a session", async () => { + const res = await createApp(null).request("/xrpc/buzz.bookhive.listPersonalShelves"); + expect(res.status).toBe(401); + }); +}); diff --git a/src/xrpc/replay-store.ts b/src/xrpc/replay-store.ts new file mode 100644 index 00000000..bc9348ff --- /dev/null +++ b/src/xrpc/replay-store.ts @@ -0,0 +1,58 @@ +/** + * Replay protection for service-auth JWTs. + * + * `ServiceJwtVerifier` calls `check({iss, jti}, ttl)` after it has verified the + * signature (deliberately — a forged token must not be able to burn store + * entries) and treats `false` as "seen before, reject". + * + * Written as a single `INSERT ... ON CONFLICT DO NOTHING` against the KV's own + * SQLite connection rather than through unstorage's get-then-set. Production + * runs four worker processes against one file, so a read followed by a write + * would let two concurrent replays of the same token both observe "unseen". + * One statement gives exactly one winner. + */ + +import { sql } from "kysely"; +import type { ReplayStore } from "@atcute/xrpc-server/auth"; +import type { KvDb } from "../sqlite-kv"; + +/** + * Its own table in the KV file, with the same `(id, value, created_at, + * updated_at)` shape every unstorage mount uses — so the existing VACUUM and + * incremental-vacuum sweeps cover it without special-casing. + */ +export const REPLAY_TABLE = "svc_jti"; + +export async function ensureReplayTable(kvDb: KvDb): Promise { + await kvDb.schema + .createTable(REPLAY_TABLE) + .ifNotExists() + .addColumn("id", "text", (col) => col.primaryKey()) + .addColumn("value", "text", (col) => col.notNull()) + .addColumn("created_at", "text", (col) => col.notNull()) + .addColumn("updated_at", "text", (col) => col.notNull()) + .execute(); +} + +export function createKvReplayStore(kvDb: KvDb): ReplayStore { + return { + async check({ iss, jti }, ttlSeconds) { + const now = new Date().toISOString(); + const expiresAt = new Date(Date.now() + ttlSeconds * 1000).toISOString(); + const result = await sql<{ inserted: number }>` + INSERT INTO ${sql.table(REPLAY_TABLE)} (id, value, created_at, updated_at) + VALUES (${`${iss}|${jti}`}, ${expiresAt}, ${now}, ${now}) + ON CONFLICT(id) DO NOTHING + RETURNING 1 as inserted + `.execute(kvDb); + return result.rows.length > 0; + }, + }; +} + +/** Drop entries whose TTL has passed. Runs on the primary worker's 15m sweep. */ +export async function sweepReplayStore(kvDb: KvDb): Promise { + await sql`DELETE FROM ${sql.table(REPLAY_TABLE)} WHERE value < ${new Date().toISOString()}`.execute( + kvDb, + ); +} diff --git a/src/xrpc/router.ts b/src/xrpc/router.ts index a61b6914..bce6daca 100644 --- a/src/xrpc/router.ts +++ b/src/xrpc/router.ts @@ -31,6 +31,9 @@ import { BuzzBookhiveGetLanguages, BuzzBookhiveGetPersonalLibrary, BuzzBookhiveGetPersonalBook, + BuzzBookhiveGetPersonalBookFile, + BuzzBookhiveGetPersonalBookCover, + BuzzBookhiveListPersonalShelves, BuzzBookhiveUploadPersonalBook, BuzzBookhiveDeletePersonalBook, BuzzBookhiveLinkPersonalBook, @@ -96,19 +99,62 @@ import type { ProfileViewDetailed, SyncProgressData, } from "../types"; -import { detectFormat, parseBook, koreaderPartialMD5 } from "../utils/bookMetadata/index"; import { - bookFilePath, - coverFilePath, - ensureDir, - MAX_PERSONAL_BOOK_BYTES, - personalBookDir, + etagMatches, + getStorageQuota, + getStorageUsage, removeBookDir, + streamPersonalBook, } from "../utils/personalLibrary"; -import { matchSyncDocument, NO_HIVE_MATCH } from "../utils/syncMatching"; +import { uploadPersonalBook, type UploadPersonalBookResult } from "../utils/uploadPersonalBook"; +import { resolveXrpcAuth, type AuthMode, type XrpcAuth, type XrpcAuthContext } from "./auth"; +import type { Nsid } from "@atcute/lexicons"; +import type { ServiceJwtVerifier } from "@atcute/xrpc-server/auth"; +import { matchSyncDocumentForUser, NO_HIVE_MATCH, SAME_BOOK_FILE } from "../utils/syncMatching"; +import { filenameKey } from "../utils/filenameMatching"; import { bridgeProgressToUserBook } from "../utils/syncBridge"; import { truncateForLog } from "../middleware/wide-event"; +/** + * The one place the upload core's failure reasons become XRPC errors, so the + * XRPC and multipart adapters can't drift on what a given failure means. The + * matching HTTP mapping lives in `src/routes/library.tsx`. + */ +function uploadErrorFor(result: Extract): XRPCError { + switch (result.reason) { + case "too-large": + return new XRPCError({ + status: 413, + error: "TooLarge", + message: `File exceeds ${result.limitBytes} bytes`, + }); + case "quota-exceeded": + return new XRPCError({ + status: 413, + error: "QuotaExceeded", + message: `Library full (${result.usedBytes} of ${result.quotaBytes} bytes used)`, + }); + case "unsupported-format": + return new InvalidRequestError({ + message: `Unsupported file format: ${result.filename}`, + }); + case "duplicate": + return new XRPCError({ + status: 409, + error: "AlreadyExists", + message: "This book already exists in your library", + }); + case "empty": + return new InvalidRequestError({ message: "The file is empty" }); + case "busy": + return new XRPCError({ + status: 503, + error: "Busy", + message: "Server is busy — try again in a moment", + }); + } +} + /** * Shape a `sync_document.progressData` blob into the lexicon's syncProgressView. * Returns undefined when the book has never been synced or the blob is unusable. @@ -130,143 +176,6 @@ function syncProgressView( } } -/** - * Process a book upload: detect format, hash, extract metadata, write to disk, - * insert DB row, and attempt auto-linking to a hive_book. Exported so it can be - * called from both the XRPC handler and a regular Hono multipart form route. - */ -export async function processBookUpload( - db: Database, - _kv: Storage, - userDid: string, - bytes: Uint8Array, - filename: string, -): Promise<{ - contentHash: string; - title: string; - authors?: string; - language?: string; - format: string; - mime: string; - sizeBytes: number; - createdAt: string; - updatedAt: string; - hiveId?: string; - coverUrl?: string; -}> { - // 1. Detect format — reject unknown - const formatInfo = detectFormat(bytes, filename); - if (formatInfo.format === "unknown") { - throw new XRPCError({ - status: 400, - error: "InvalidRequest", - message: `Unsupported file format: ${filename}`, - }); - } - - // 2. Compute content hash - const contentHash = koreaderPartialMD5(bytes); - - // 3. Check for duplicate - const duplicate = await db - .selectFrom("personal_book") - .select(["id", "contentHash"]) - .where("userDid", "=", userDid) - .where("contentHash", "=", contentHash) - .executeTakeFirst(); - - if (duplicate) { - throw new XRPCError({ - status: 409, - error: "AlreadyExists", - message: "This book already exists in your library", - }); - } - - // 4. Parse metadata + cover - const metadata = parseBook(bytes, filename); - - // 5. Write file to disk - const dir = personalBookDir(userDid, contentHash); - await ensureDir(dir); - const filePath = bookFilePath(userDid, contentHash, formatInfo.ext); - await Bun.write(filePath, bytes); - - // 6. Write cover if extracted - let coverPath: string | null = null; - let coverMime: string | null = null; - if (metadata.cover) { - coverPath = coverFilePath(userDid, contentHash, metadata.cover.ext); - coverMime = metadata.cover.mime; - await Bun.write(coverPath, metadata.cover.bytes); - } - - // 7. Insert into personal_book - const now = new Date().toISOString(); - - // 8. Auto-link: try to match to a hive_book - let hiveId = await matchSyncDocument(db, { - title: metadata.title, - authors: metadata.authors, - filename, - }); - - await db - .insertInto("personal_book") - .values({ - userDid, - contentHash, - hiveId, - filename, - title: metadata.title, - authors: metadata.authors || null, - language: metadata.language ?? null, - format: formatInfo.format, - mime: formatInfo.mime, - filePath, - coverPath, - coverMime, - sizeBytes: bytes.length, - createdAt: now, - updatedAt: now, - }) - .execute(); - - // 9. If contentHash matches a sync_document for this user, update its hiveId - if (hiveId) { - await db - .updateTable("sync_document") - .set({ hiveId }) - .where("userDid", "=", userDid) - .where("documentHash", "=", contentHash) - .where("hiveId", "is", null) - .execute(); - - // Mark the book as owned if the user has it in their library - await db - .updateTable("user_book") - .set({ owned: 1 }) - .where("userDid", "=", userDid) - .where("hiveId", "=", hiveId) - .where("owned", "=", 0) - .execute(); - } - - return { - contentHash, - title: metadata.title, - authors: metadata.authors || undefined, - language: metadata.language ?? undefined, - format: formatInfo.format, - mime: formatInfo.mime, - sizeBytes: bytes.length, - createdAt: now, - updatedAt: now, - hiveId: hiveId ?? undefined, - coverUrl: coverPath ? `/library/covers/${contentHash}` : undefined, - }; -} - /** Minimal context shape required by XRPC handlers (avoids importing index). */ export type XrpcContext = { db: Database; @@ -279,6 +188,10 @@ export type XrpcContext = { handle: { resolve: (handle: string) => Promise }; }; addWideEventContext: (context: Record) => void; + /** Verifies atproto service-auth JWTs. Null when service auth is disabled. */ + serviceJwtVerifier?: ServiceJwtVerifier | null; + /** Gate on service auth: has this DID ever used BookHive? */ + isKnownAccount?: (did: string) => Promise; }; export type XrpcDeps = { @@ -298,27 +211,78 @@ function getCtx(): XrpcContext { return ctx; } +/** + * Auth resolved for the in-flight handler, by the registration wrapper below. + * Same AsyncLocalStorage idiom as the context — atcute handlers only receive + * `{request, params, input, signal}`, so there is nowhere else to put it. + */ +const xrpcAuthStorage = new AsyncLocalStorage(); + +/** The authenticated caller. Only valid inside a handler registered with `auth`. */ +function getAuth(): XrpcAuth { + const auth = xrpcAuthStorage.getStore(); + if (!auth) throw new Error("XRPC auth not resolved (method registered without `auth`)"); + return auth; +} + +/** + * The caller's OAuth session, for handlers that write to their repo. Non-null by + * construction — `auth: "pdsWrite"` refuses service auth before the handler runs + * — but narrowing the union keeps that guarantee in the types rather than in a + * comment. + */ +function requireAgent(): SessionClient { + const auth = getAuth(); + if (auth.method !== "session") { + throw new AuthRequiredError({ message: "This method requires an OAuth session" }); + } + return auth.agent; +} + export function createXrpcRouter( app: import("hono").Hono<{ Variables: V }>, deps: XrpcDeps, ): void { const router = new XRPCRouter(); - // XRPCRouter catches handler throws and turns them into a 500 Response, so - // Hono's error-capture middleware never sees them and the wide event logs an - // error-level line with no `error` field at all. Record the cause on the way - // past. Patching the two registration methods once beats annotating 40+ - // handlers (and can't be forgotten by the next one added). + // Two things are patched onto every registration here rather than repeated in + // 40+ handlers (where the next one added would forget them): + // + // 1. **Error observability.** XRPCRouter catches handler throws and turns them + // into a 500 Response, so Hono's error-capture middleware never sees them + // and the wide event logs an error-level line with no `error` field at all. + // Record the cause on the way past. + // 2. **Authentication**, when the registration carries an `auth` mode. The + // `lxm` a service-auth token must be bound to is derived from the schema's + // own NSID, which makes it structurally impossible for a method's route and + // its token binding to disagree. for (const method of ["addQuery", "addProcedure"] as const) { const original = router[method].bind(router) as (schema: unknown, options: any) => unknown; - (router as any)[method] = (schema: unknown, options: any) => { + (router as any)[method] = (schema: any, options: any) => { const handler = options?.handler; if (typeof handler !== "function") return original(schema, options); + + // A generated lexicon module carries `mainSchema`; `v.query`/`v.procedure` + // put the NSID on it. Same unwrap atcute does internally. + const nsid = ("mainSchema" in schema ? schema.mainSchema : schema).nsid as Nsid; + const mode: AuthMode | undefined = options.auth; + const { auth: _auth, ...rest } = options; + return original(schema, { - ...options, - handler: async (input: unknown) => { + ...rest, + handler: async (input: any) => { try { - return await handler(input); + if (mode === undefined) return await handler(input); + + const ctx = xrpcContextStorage.getStore(); + const auth = await resolveXrpcAuth(ctx as XrpcAuthContext, input.request, { + lxm: nsid, + mode, + }); + ctx?.addWideEventContext({ userDid: auth.did, xrpc_auth: auth.method }); + // Auth failures land inside this try, so a 401 is recorded as the + // intentional control flow it is — same as a hand-thrown one. + return await xrpcAuthStorage.run(auth, () => handler(input)); } catch (err) { // Deliberate 4xx (AuthRequiredError, InvalidRequest, …) are control // flow, not defects — record them without a stack. @@ -1272,10 +1236,10 @@ export function createXrpcRouter - join - .onRef("sync_document.documentHash", "=", "personal_book.contentHash") - .on("sync_document.userDid", "=", userDid) - .on("sync_document.provider", "=", "kosync"), - ) .select([ "personal_book.id", "personal_book.contentHash", @@ -1496,11 +1450,56 @@ export function createXrpcRouter [ + eb + .selectFrom("sync_document") + .select("sync_document.progressData") + .where("sync_document.userDid", "=", userDid) + .where("sync_document.provider", "=", "kosync") + .where(SAME_BOOK_FILE) + .orderBy("sync_document.updatedAt", "desc") + .limit(1) + .as("progressData"), + eb + .selectFrom("sync_document") + .select("sync_document.updatedAt") + .where("sync_document.userDid", "=", userDid) + .where("sync_document.provider", "=", "kosync") + .where(SAME_BOOK_FILE) + .orderBy("sync_document.updatedAt", "desc") + .limit(1) + .as("progressUpdatedAt"), + ]) + .where("personal_book.userDid", "=", userDid); + + // Same predicate and ordering as the OPDS search feed, so "full parity" + // is a property of the SQL rather than a claim. SQLite's LIKE is + // case-insensitive for ASCII only; that is pre-existing OPDS behaviour + // and deliberately preserved rather than silently changed here. + if (q) { + query = query.where((eb) => + eb.or([ + eb("personal_book.title", "like", `%${q}%`), + eb("personal_book.authors", "like", `%${q}%`), + ]), + ) as typeof query; + } + query = + sort === "title" + ? (query.orderBy("personal_book.title", "asc") as typeof query) + : sort === "author" + ? (query + .orderBy("personal_book.authors", "asc") + .orderBy("personal_book.title", "asc") as typeof query) + : (query.orderBy("personal_book.createdAt", "desc") as typeof query); if (shelfId !== undefined) { query = query @@ -1518,6 +1517,14 @@ export function createXrpcRouter eb.fn.countAll().as("total")) .where("personal_book.userDid", "=", userDid); + if (q) { + countQuery = countQuery.where((eb) => + eb.or([ + eb("personal_book.title", "like", `%${q}%`), + eb("personal_book.authors", "like", `%${q}%`), + ]), + ) as typeof countQuery; + } if (shelfId !== undefined) { countQuery = countQuery .innerJoin( @@ -1528,12 +1535,16 @@ export function createXrpcRouter MAX_PERSONAL_BOOK_BYTES) { - throw new InvalidRequestError({ message: "File exceeds 100 MB limit" }); + // The XRPC equivalent of GET /opds/books/:hash/cover. + router.addQuery(BuzzBookhiveGetPersonalBookCover, { + auth: "identity", + async handler({ request, params: _params }) { + const ctx = getCtx(); + const { did: userDid } = getAuth(); + const { contentHash, width = 300 } = _params as BuzzBookhiveGetPersonalBookCover.$params; + + const book = await ctx.db + .selectFrom("personal_book") + .select(["coverPath", "coverMime", "hiveId"]) + .where("userDid", "=", userDid) + .where("contentHash", "=", contentHash) + .executeTakeFirst(); + if (!book) { + throw new XRPCError({ status: 404, error: "NotFound", message: "Book not found" }); } - const bytes = new Uint8Array(await request.arrayBuffer()); - if (bytes.length > MAX_PERSONAL_BOOK_BYTES) { - throw new InvalidRequestError({ message: "File exceeds 100 MB limit" }); + if (book.coverPath) { + const file = Bun.file(book.coverPath); + if (await file.exists()) { + // Set our own ETag: hono's `etag()` only digests (and so buffers) a + // response that doesn't already carry one, and this gets conditional + // requests answered for free. + const etag = `"${contentHash}-cover"`; + if (etagMatches(request.headers.get("if-none-match"), etag)) { + return new Response(null, { status: 304, headers: { ETag: etag } }); + } + return new Response(file.stream(), { + headers: { + "Content-Type": book.coverMime || "image/jpeg", + "Content-Length": String(file.size), + "Cache-Control": "private, max-age=86400", + ETag: etag, + }, + }); + } } - const filename = request.headers.get("x-file-name") ?? "unknown"; - const result = await processBookUpload(ctx.db, ctx.kv, userDid, bytes, filename); - return json({ book: result }); + // No extracted cover, but the book is linked to a catalog entry: hand the + // client the public image proxy. Absolute, so a non-browser client can + // follow it without knowing our origin, and public, so nothing leaks. + if (book.hiveId) { + return Response.redirect( + new URL(`/images/books/${book.hiveId}?w=${width}`, request.url).toString(), + 302, + ); + } + throw new XRPCError({ status: 404, error: "NotFound", message: "No cover for this book" }); + }, + }); + + // The root call for a catalog client: everything GET /opds renders, in one + // request — shelves with their counts, the library total, and storage usage. + router.addQuery(BuzzBookhiveListPersonalShelves, { + auth: "identity", + async handler() { + const ctx = getCtx(); + const { did: userDid } = getAuth(); + + const [shelves, counted, usedBytes] = await Promise.all([ + ctx.db + .selectFrom("personal_shelf") + .leftJoin("personal_shelf_item", "personal_shelf.id", "personal_shelf_item.shelfId") + .select((eb) => [ + "personal_shelf.id", + "personal_shelf.name", + "personal_shelf.description", + "personal_shelf.createdAt", + "personal_shelf.updatedAt", + eb.fn.count("personal_shelf_item.personalBookId").as("bookCount"), + ]) + .where("personal_shelf.userDid", "=", userDid) + .groupBy("personal_shelf.id") + .orderBy("personal_shelf.name", "asc") + .execute(), + ctx.db + .selectFrom("personal_book") + .select((eb) => eb.fn.countAll().as("total")) + .where("userDid", "=", userDid) + .executeTakeFirstOrThrow(), + getStorageUsage(ctx.db, userDid), + ]); + + return json({ + shelves: shelves.map((s) => ({ + id: s.id, + name: s.name, + description: s.description ?? undefined, + bookCount: Number(s.bookCount), + createdAt: s.createdAt, + updatedAt: s.updatedAt, + })), + totalBooks: Number(counted.total), + storage: { usedBytes, quotaBytes: getStorageQuota() }, + }); + }, + }); + + // The blob-input twin of POST /library/upload. Both are thin adapters over + // `uploadPersonalBook`; the body streams straight to disk from here, so an + // oversized or malformed upload is never materialised in memory. + router.addProcedure(BuzzBookhiveUploadPersonalBook, { + auth: "identity", + async handler({ request, params: _params }) { + const ctx = getCtx(); + const { did: userDid } = getAuth(); + const { filename } = _params as BuzzBookhiveUploadPersonalBook.$params; + + const declared = Number(request.headers.get("content-length")); + const result = await uploadPersonalBook({ + db: ctx.db, + kv: ctx.kv, + userDid, + filename, + source: { + kind: "stream", + // The lexicon declares a blob input, so atcute leaves the body alone + // and types it as a stream for us. + body: request.body as ReadableStream, + declaredLength: Number.isFinite(declared) && declared > 0 ? declared : undefined, + }, + }); + + if (!result.ok) throw uploadErrorFor(result); + return json({ + book: result.book, + storageUsedBytes: result.storageUsedBytes, + storageQuotaBytes: result.storageQuotaBytes, + }); }, }); router.addProcedure(BuzzBookhiveDeletePersonalBook, { + auth: "identity", async handler({ input: _input }) { const ctx = getCtx(); - const agent = await ctx.getSessionAgent(); - if (!agent) throw new AuthRequiredError({ message: "Authentication required" }); - const userDid = agent.did; + const { did: userDid } = getAuth(); const { contentHash } = _input as BuzzBookhiveDeletePersonalBook.$input; const book = await ctx.db @@ -1704,18 +1855,26 @@ export function createXrpcRouter Date: Tue, 11 Aug 2026 22:50:42 +0200 Subject: [PATCH 2/9] fix: apply review findings, rasterize SVG covers, correct AGENTS.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the personal-library XRPC work, plus two things found while testing it against a real book. Review fixes: - isKnownAccount is required on XrpcAuthContext/XrpcContext and called unconditionally. It was optional and guarded by truthiness, so a context that merely forgot to wire it opened the personal library to any DID on the network. - getPersonalBookCover builds its 302 by hand rather than with Response.redirect, whose headers are immutable — the downstream Cache-Control middleware mutating them throws a TypeError and turns the redirect into a 500. - Every getPersonalLibrary sort ends on personal_book.id. None of the leading keys are unique, so LIMIT/OFFSET pages could repeat one book and drop another. - insertIfUnderQuota gained ON CONFLICT DO NOTHING and tells a duplicate apart from a quota rejection, instead of raising a UNIQUE violation when the same file races past the duplicate check. - A failed cover write clears coverPath instead of propagating, so a committed row can never point at a file that isn't there. - The replay sweep is gated on XRPC_SERVICE_AUTH as well as ..._REPLAY; the svc_jti table only exists when both are on. - syncMatching: the hiveId-null guard is enforced in the UPDATE, and author FTS queries can no longer starve the title query out of all four slots. - epub/cbz re-check inflated cover bytes against MAX_COVER_BYTES; the ZIP's declared originalSize is attacker-controlled and covers bypass the quota. - LibraryManager refetches after a delete so the storage meter shows freed space; GET /library validates ?error= against a closed set. SVG covers (image-meta + @resvg/resvg-js): Standard Ebooks ships every cover as an SVG holding the artwork in an element plus the title as ~40 vector paths. Bun.Image rejects SVG as an "unrecognised format", so that whole corpus uploaded with no cover at all. prepareCover now rasterizes via resvg and encodes to JPEG, inside the parse semaphore so it shares the upload's native-memory bound. Both shortcuts were tried and each loses half the cover: unwrapping the embedded raster drops the lettering, rendering with @takumi-rs drops the artwork. @resvg/resvg-js* is in traceDeps because a missing native binding does not crash — it silently restores the bug. Also fixes PNG_32, which was never a valid PNG: it built IDAT with Bun.deflateSync (raw deflate) where the format requires zlib. Bun's decoder is lenient, resvg's is not, so the fixture only worked by accident. AGENTS.md, audited against the codebase: seed:db documented a script whose entrypoint was deleted long ago, 44 lexicons not ~27, 5 worker bundles not 4, personal_book/personal_shelf_item key columns, the @ alias (configured in Vite but absent from tsconfig, so unusable in src/), and the unreferenced screenshot count. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 92 ++++++++++++---- bun.lock | 30 ++++++ lexicons/getPersonalLibrary.json | 3 +- lexicons/uploadPersonalBook.json | 2 +- package.json | 2 + .../types/buzz/bookhive/getPersonalLibrary.ts | 7 +- .../types/buzz/bookhive/uploadPersonalBook.ts | 4 +- src/client/components/LibraryManager.tsx | 49 ++++----- src/context.ts | 5 +- src/db.ts | 5 + src/routes/library.tsx | 92 ++++++++++------ src/utils/bookMetadata/bookMetadata.test.ts | 92 ++++++++++++++-- src/utils/bookMetadata/cbz.ts | 5 +- src/utils/bookMetadata/cover.ts | 102 ++++++++++++++++-- src/utils/bookMetadata/epub.ts | 6 +- src/utils/bookMetadata/index.ts | 2 +- src/utils/bookMetadata/testFixtures.ts | 91 ++++++++++++++-- src/utils/catalogBookService.ts | 5 +- src/utils/filenameMatching.ts | 14 +++ src/utils/imageProxy.ts | 7 +- src/utils/syncMatching.ts | 34 ++++-- src/utils/uploadPersonalBook.test.ts | 58 +++++++++- src/utils/uploadPersonalBook.ts | 84 +++++++++++---- src/xrpc/auth.ts | 10 +- src/xrpc/personalLibrary.test.ts | 47 +++++++- src/xrpc/router.ts | 40 +++++-- vite.config.ts | 17 ++- 27 files changed, 745 insertions(+), 160 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 717d3e7a..fb8bccf2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,17 +41,18 @@ Browser ──> Bun.serve() ──> Hono app ──> Server-rendered JSX pages │ └── Worker threads (see below) └── static files (public/) -Worker threads (src/workers/, bundled to .output/server/workers/): - ingester-worker — Jetstream firehose ingest - og-render-worker — OG image generation (React + takumi) - open-observe-worker — pino log shipping to OpenObserve - import-worker — CSV import processing +Worker threads (bundled to .output/server/workers/): + ingester-worker — Jetstream firehose ingest (src/workers/) + og-render-worker — OG image generation (React+takumi) (src/workers/) + open-observe-worker — pino log shipping to OpenObserve (src/workers/) + import-worker — CSV import processing (src/workers/) + waf-solver-worker — AWS WAF challenge solve (src/scrapers/waf/) ``` **Key patterns:** - Server components (`src/pages/`) render full HTML. Only 6 islands are hydrated client-side (`src/client/`). Most interactivity is CSS-only (peer/checked selectors) or inline `