diff --git a/apps/api/test/server.test.ts b/apps/api/test/server.test.ts
index ffab4f9..d9cac07 100644
--- a/apps/api/test/server.test.ts
+++ b/apps/api/test/server.test.ts
@@ -145,6 +145,7 @@ function stubCore(
createApiKey: async () => ({ key: "bnb_secret", record: sampleApiKey }),
listApiKeys: async () => [sampleApiKey],
revokeApiKey: async () => true,
+ findByUsername: async () => null,
...authOverrides,
},
settingsService: {
diff --git a/apps/web/src/plugins/shimmie-import/ShimmieImportSection.tsx b/apps/web/src/plugins/shimmie-import/ShimmieImportSection.tsx
index e4d96bd..e73f6e9 100644
--- a/apps/web/src/plugins/shimmie-import/ShimmieImportSection.tsx
+++ b/apps/web/src/plugins/shimmie-import/ShimmieImportSection.tsx
@@ -1,5 +1,6 @@
import { useEffect, useRef, useState, type FormEvent } from "react";
+import { useQuery, useQueryClient } from "@tanstack/react-query";
import { Loader2 } from "lucide-react";
import { shimmieImportApi } from "./client";
@@ -38,15 +39,26 @@ export function ShimmieImportSection() {
const [apiKey, setApiKey] = useState("");
const [allUsers, setAllUsers] = useState(true);
const [usersList, setUsersList] = useState("");
- const [targetUserId, setTargetUserId] = useState("");
+ const [targetUsername, setTargetUsername] = useState("");
const [sourceTimezone, setSourceTimezone] = useState("UTC");
- const [busy, setBusy] = useState<"idle" | "preflight" | "importing">("idle");
+ const [busy, setBusy] = useState<"idle" | "preflight" | "importing" | "retrying">("idle");
const [preflight, setPreflight] = useState<{ actingUser: string; maxId: number } | null>(null);
const [progress, setProgress] = useState(null);
+ const [runId, setRunId] = useState(null);
const [error, setError] = useState(null);
const [notice, setNotice] = useState(null);
const cancelRef = useRef(false);
+ // The run the progress panel currently describes — read via ref so `onRetry`'s
+ // async loop can't act on a stale closure when deciding whether to update it.
+ const activeRunIdRef = useRef(null);
+
+ const queryClient = useQueryClient();
+ const runsQuery = useQuery({
+ queryKey: ["shimmie-import", "runs"],
+ queryFn: async () => call(await importApi.runs.get()),
+ });
+ const refreshRuns = () => queryClient.invalidateQueries({ queryKey: ["shimmie-import", "runs"] });
// Stop the import loop if the component unmounts (leaving /admin), so it doesn't
// keep hitting the API or setting state on an unmounted component.
@@ -93,17 +105,12 @@ export function ShimmieImportSection() {
setError("List at least one shimmie username, or choose “all users”.");
return;
}
- const targetId = targetUserId.trim() ? Number(targetUserId.trim()) : undefined;
- if (targetId !== undefined && (!Number.isInteger(targetId) || targetId < 1)) {
- setError("Target user id must be a positive integer (or blank for yourself).");
- return;
- }
const started = await call(
await importApi.runs.post({
baseUrl,
apiKey,
users,
- targetUserId: targetId,
+ targetUsername: targetUsername.trim() || undefined,
sourceTimezone: sourceTimezone.trim() || "UTC",
}),
);
@@ -111,14 +118,16 @@ export function ShimmieImportSection() {
setError(started.error);
return;
}
- const runId = started.runId;
+ const activeRunId = started.runId;
+ setRunId(activeRunId);
+ activeRunIdRef.current = activeRunId;
for (;;) {
if (cancelRef.current) {
- await importApi.runs({ id: runId }).cancel.post();
+ await importApi.runs({ id: activeRunId }).cancel.post();
setNotice("Import canceled.");
break;
}
- const step = await call(await importApi.runs({ id: runId }).step.post({ apiKey }));
+ const step = await call(await importApi.runs({ id: activeRunId }).step.post({ apiKey }));
// Bail after the await if we were unmounted / canceled mid-request.
if (cancelRef.current) break;
if (!step.ok) {
@@ -142,6 +151,51 @@ export function ShimmieImportSection() {
setError("Import request failed (are you signed in as an admin?).");
} finally {
setBusy("idle");
+ void refreshRuns();
+ }
+ }
+
+ async function onRetry(targetRunId: number) {
+ if (busy !== "idle") return;
+ if (!apiKey) {
+ setError("Enter the shimmie API key to retry.");
+ return;
+ }
+ resetMessages();
+ setBusy("retrying");
+ cancelRef.current = false;
+ try {
+ for (;;) {
+ if (cancelRef.current) break;
+ const res = await call(await importApi.runs({ id: targetRunId })["retry-failed"].post({ apiKey }));
+ if (cancelRef.current) break;
+ if (!res.ok) {
+ setError(res.error);
+ break;
+ }
+ // Only touch the progress panel when retrying the run it describes — a
+ // history-row retry of a different run must not overwrite it.
+ if (targetRunId === activeRunIdRef.current) {
+ setProgress((prev) =>
+ prev ? { ...prev, imported: prev.imported + res.recovered, failed: res.remainingFailed } : prev,
+ );
+ }
+ // Stop when nothing's left OR this batch made no progress — otherwise a
+ // set of permanently-failing posts (re-selected each call) would loop forever.
+ if (res.remainingFailed === 0 || res.recovered === 0) {
+ setNotice(
+ res.remainingFailed === 0
+ ? "Retry complete — all recovered."
+ : `Retry stopped · ${res.remainingFailed} still failing.`,
+ );
+ break;
+ }
+ }
+ } catch {
+ setError("Retry request failed (are you signed in as an admin?).");
+ } finally {
+ setBusy("idle");
+ void refreshRuns();
}
}
@@ -216,13 +270,13 @@ export function ShimmieImportSection() {
- Attribute to user id
+ Attribute to bunbooru user
setTargetUserId(e.target.value)}
- placeholder="blank = you"
+ type="text"
+ value={targetUsername}
+ onChange={(e) => setTargetUsername(e.target.value)}
+ placeholder="username (blank = you)"
+ autoComplete="off"
className={INPUT_CLASS}
/>
@@ -267,7 +321,7 @@ export function ShimmieImportSection() {
{busy === "importing" ? : null}
Import
- {busy === "importing" ? (
+ {busy === "importing" || busy === "retrying" ? (
(cancelRef.current = true)}
@@ -297,6 +351,59 @@ export function ShimmieImportSection() {
{progress.failed > 0 ? ` · ${progress.failed} failed` : ""}
{progress.done ? " · done" : ""}
+ {progress.failed > 0 && runId !== null ? (
+ void onRetry(runId)}
+ disabled={busy !== "idle" || !apiKey}
+ className="mt-1 flex items-center gap-1 rounded border border-line px-3 py-1.5 disabled:cursor-not-allowed disabled:opacity-60"
+ >
+ {busy === "retrying" ? : null}
+ Retry {progress.failed} failed
+
+ ) : null}
+
+ ) : null}
+
+ {runsQuery.data && runsQuery.data.length > 0 ? (
+
+
Recent runs
+
+ {runsQuery.data.map((run) => (
+
+
+ #{run.id} · {run.sourceInstance} · {run.userFilter === "*" ? "all users" : run.userFilter}
+
+
+
+ {run.status} · {run.imported}
+ ✓
+ imported
+ {run.failed > 0 ? (
+ <>
+ {" · "}
+ {run.failed}
+ ✗
+ failed
+ >
+ ) : null}{" "}
+ · {run.cursor}/{run.maxId}
+
+ {run.failed > 0 && run.status !== "canceled" ? (
+ void onRetry(run.id)}
+ disabled={busy !== "idle" || !apiKey}
+ title={apiKey ? "Retry this run's failed posts" : "Enter the API key above to retry"}
+ className="rounded border border-line px-2 py-0.5 text-link disabled:cursor-not-allowed disabled:opacity-60"
+ >
+ Retry
+
+ ) : null}
+
+
+ ))}
+
) : null}
diff --git a/docs/ingest-clients.md b/docs/ingest-clients.md
new file mode 100644
index 0000000..7abcc93
--- /dev/null
+++ b/docs/ingest-clients.md
@@ -0,0 +1,380 @@
+# Ingest Clients — Design Plan
+
+**Status:** draft, not implemented. Revisit before building.
+
+Companion software that gets media *into* Bunbooru from where users actually
+find it: the desktop browser, the mobile share sheet, and the phone's camera
+roll. Server-side hook design lives in [ingest-hooks.md](./ingest-hooks.md);
+this doc covers the clients and the API surface they need.
+
+---
+
+## 1. Goal
+
+Bunbooru is an image (later video) **backup and library server**. Capture must
+be as close to one gesture as possible:
+
+- **Desktop:** right-click an image → "Upload to Bunbooru" → done.
+- **Mobile ad-hoc:** share sheet → Bunbooru → done.
+- **Mobile bulk:** camera roll syncs in the background, no gesture at all.
+
+### The friction rule
+
+**Tagging must never be required at capture time.** Every quick-capture flow
+dies on "now enter tags." Upload lands untagged into a *needs tagging* queue;
+triage happens later in bulk from the web UI. Clients may *offer* tags
+(the extension popup should), but must never block on them.
+
+Content-hash dedupe (already in `asset-service`) makes this safe: re-sharing
+the same image is idempotent, so users can be sloppy. Clients should surface
+"already in your library" rather than an error.
+
+---
+
+## 2. Server-side prerequisites
+
+Almost all of this is already built. What's missing:
+
+| Need | Status |
+|---|---|
+| One-shot upload (`POST /assets`, multipart, dedupes on sha256) | **exists** |
+| Resumable chunked upload (`POST/GET/DELETE /uploads[/:token]`) | **exists** |
+| API keys | **exists** (`bnb_…`, sha256-stored) |
+| Upload-scoped API key permission | **needed** |
+| `POST /assets/exists` batch hash check | **needed** — see hooks doc §10 |
+| `findManyBySha256` on the asset repository | **needed** |
+
+`POST /assets/exists` returns four states per hash — `present | absent |
+deleted | banned`, resolved by the precedence rule in hooks doc §10 — and is
+queried against `originalSha256` once transforms exist. It is a batch hash
+oracle, so it requires auth, rate-limiting, and a decision on whether `deleted`
+and `banned` are owner-scoped or globally visible (hooks doc §10, open
+question 6). That scoping must be settled before the endpoint ships: an
+upload-scoped key should not become a way to enumerate moderation state.
+
+### What clients hash
+
+**Clients hash the original bytes as they exist on the device** — the same bytes
+they would upload — and never the server's post-transform payload, which they
+cannot compute. Existence checks therefore resolve against `originalSha256`
+while `assets.sha256` continues to drive storage keys and dedupe.
+
+The awkward case is many-to-one: several originals differing only in metadata
+can transform to a single stored asset. The server must record **every**
+received-byte hash against the resulting asset and return that asset's canonical
+id, so a client whose variant maps onto an existing asset marks it synced rather
+than re-uploading it forever. Whether that is a column or a side table is hooks
+doc §11 open question 1, and it must be resolved before this endpoint is built.
+
+### API key handling (all clients)
+
+Every client here authenticates with a bearer `bnb_…` key. Shared rules:
+
+- **Upload-only scope.** An ingest client never needs read, delete, or admin.
+ A leaked key should be able to add content, not exfiltrate or destroy it.
+- **One key per device, independently revocable and rotatable**, surfaced in
+ account settings as named devices ("Pixel 9", "work laptop"). Losing a phone
+ must not mean rotating every client's key. The per-device model is
+ open question 5 and should be settled before the first client ships, since it
+ shapes the API-key schema.
+- **Secure storage:** OS keychain / Keystore on mobile, `chrome.storage` on the
+ extension (never `localStorage` on a page, never a committed config file).
+- **Redact from logs and errors** — including client-side crash reports and
+ server request logs. A key in a stack trace is a leaked key.
+- **Never embed a key in an exported or synced Shortcut.** iCloud syncs
+ Shortcuts across devices and users share `.shortcut` files freely; a shared
+ template must prompt for the key on first run and store it in a Shortcuts
+ text field the user fills, not ship one baked in. Call this out explicitly in
+ the §6 template's documentation.
+
+---
+
+## 3. Browser extension (MV3)
+
+The primary desktop surface, and the one that motivated this work.
+
+### Decision: the extension uploads bytes, not a URL
+
+Rejected alternative: a server-side `POST /assets/from-url`. It sounds simpler
+but immediately fights the target site's auth, hotlink protection, Referer
+checks, and signed/expiring URLs.
+
+An extension's `fetch()` with `host_permissions` **bypasses CORS and sends the
+site's cookies**, so the exact cases that defeat server-side fetching
+(Pixiv, Twitter, Patreon, anything gated) are the cases the extension handles
+trivially. The image already rendered, so the fetch is normally an HTTP cache
+hit — no second download.
+
+`from-url` stays on the roadmap, but as the *server-side ingest* path for
+watch-folder / gallery-dl / bot plugins — not as the extension's mechanism.
+
+### Decision: `Blob` + `FormData`, never base64
+
+Base64 costs +33% on the wire, a full copy of the image as a JS string, and a
+decode on the server. Unnecessary:
+
+```js
+const blob = await (await fetch(imgUrl, { credentials: "include" })).blob();
+const fd = new FormData();
+fd.append("file", blob, filename);
+fd.append("source", pageUrl);
+// → POST /assets, unchanged
+```
+
+This posts to the **existing** `POST /assets` — it already takes multipart
+`t.File()`, already dedupes, already returns 200-vs-201. **The extension needs
+zero new API surface.**
+
+The reason base64 seems necessary is that `chrome.runtime.sendMessage` is
+JSON-serialized and cannot carry a `Blob` between content script and service
+worker. The fix is not to encode the bytes — it is to **never move bytes across
+that boundary**: the content script sends the *URL*, the service worker does
+both the fetch and the upload. Bytes are created and consumed in one context.
+
+### That fetch is credentialed — gate it
+
+`credentials: "include"` sends the user's cookies for the target origin, and the
+URL arrives over `chrome.runtime` messaging. Treat every such message as
+untrusted input; an arbitrary content script must not be able to make the
+extension issue an authenticated request and ship the response to Bunbooru.
+
+Required before credentials are attached:
+
+- **An explicit user gesture** — context-menu click or popup action. Never a
+ page-initiated message.
+- **Validate the sender** (`sender.tab`, `sender.origin`) and confirm it matches
+ the tab the gesture came from.
+- **Allow only `https:` (and `http:` for the user's own LAN server), plus
+ `blob:`/`data:` already owned by the page.** Reject `file:`, `chrome:`,
+ `chrome-extension:`, and anything else outright.
+- **Confirm the origin is within the extension's granted `host_permissions`.**
+
+Credentials are attached only after all four pass. On failure, fall back to an
+uncredentialed fetch rather than refusing outright — most images need no cookies.
+
+### Features
+
+- Context menu on images/video/links → "Upload to Bunbooru"
+- Popup: optional tags with autocomplete against the existing tag endpoint,
+ prefilled source URL + page title
+- "Upload all images on this page" for gallery pages
+- Options: server URL + API key
+
+### Constraints to design around
+
+**MV3 service workers get killed** (~30s idle, hard cap ~5min). Fine for
+images; not for planned video. Anything large must go through the resumable
+chunk endpoints so a killed worker resumes rather than restarts.
+
+Resuming only works if the state outlives the worker, so **upload state lives in
+`chrome.storage`, not in worker memory**: file identity (the source URL plus its
+hash once known), the upload token, the last *server-confirmed* offset, and a
+retry count. On startup the worker reconciles any in-flight entries before
+accepting new work.
+
+Chunk writes and finalization must be **idempotent and safely retryable** — a
+worker can die between "server committed the chunk" and "client recorded the
+offset," so replaying a chunk at an already-committed offset must succeed rather
+than corrupt or double-append. Never advance the local offset from the request
+that was *sent*; advance it only from the offset the server *returns*
+(`GET /uploads/:token` is the authority).
+
+Two failure modes need explicit handling: **abandoned server sessions** (the
+staged upload expired or was cancelled — detect on 404/409 and restart the
+upload from zero) and **permanently failed queue entries** (retry budget
+exhausted — surface them in the popup as actionable rather than retrying
+forever). The same applies to the PWA's Background Sync queue in §5.
+
+**When `fetch` genuinely fails** (rare — per-session signed URLs), do *not*
+fall back to canvas: cross-origin images without CORS headers taint the canvas
+and `toBlob` throws. Fall back to opening the upload page with the URL
+prefilled for manual drag-drop.
+
+---
+
+## 4. Web app improvements
+
+Small, cheap, independently useful:
+
+- **Global paste handler** — `Ctrl+V` an image anywhere queues it. Covers
+ screenshots, which no URL-based flow can.
+- **Drag-and-drop** anywhere, not only on `/upload`.
+- **Needs-tagging queue** — the triage view the friction rule depends on.
+
+---
+
+## 5. PWA share target (Android stopgap)
+
+A `share_target` entry in the web manifest puts Bunbooru in the Android system
+share sheet with **zero app-store involvement** — share from Chrome, Gallery,
+or Telegram and it lands in the upload queue.
+
+- Cost: a manifest entry plus one POST route. An afternoon.
+- Add Background Sync so a share on bad signal still lands later.
+- **iOS Safari does not support `share_target`.** This is Android-only, by
+ platform limitation, not by choice.
+
+Ships *before* the mobile app so mobile capture works while the app is built.
+Nothing here is thrown away afterward.
+
+---
+
+## 6. iOS Shortcut (interim)
+
+Until the app exists, a share-sheet Shortcut hitting the API with an API key
+covers iOS. Costs **no code** — a documented shortcut template users import
+once. Same workaround Immich and Karakeep ship.
+
+---
+
+## 7. Mobile app
+
+One app covering **share target + gallery sync**. This is the only way to get a
+share extension on iOS at all, and the only way to get real background backup.
+It is by far the largest item in this plan.
+
+### Gallery sync is mostly already built
+
+`sha256` is a unique content key with `findBySha256` in the repository, so sync
+reduces to a stateless diff:
+
+1. Enumerate the camera roll
+2. Hash anything not in the local hash cache
+3. Ask the server which hashes are missing (`POST /assets/exists`)
+4. Upload only those
+
+**This is stateless on purpose.** There is no "last synced" cursor to corrupt
+or reset. Reinstall the app, restore the phone from backup, switch devices —
+the diff recomputes from content and self-heals. Cursor-based sync breaks in
+all three cases.
+
+The expensive part is hashing, not uploading. `POST /assets/exists` turns a
+20,000-photo first sync from ~80GB of uploads into ~40 requests of ~32KB.
+
+### Local database
+
+**A cache, never a source of truth.** Every inconsistency resolves by re-running
+the hash diff — that property is what makes the design self-healing, and any
+optimization that makes the local DB authoritative must be rejected.
+
+```text
+localId, mtime, sizeBytes, sha256,
+state, -- discovered | hashed | uploaded | skipped_deleted
+ -- | skipped_banned | failed
+serverAssetId,
+lastVerifiedAt, attemptCount, lastError
+```
+
+Its real job is **avoiding rehashing**. Key on `(localId, mtime, size)`; if any
+change, rehash — iOS edits create a new version of the asset in place.
+
+Identity gotchas:
+
+- **Android `MediaStore._ID` is not stable** across media rescans.
+- **iOS `PHAsset.localIdentifier` is stable until restore-from-backup**, which
+ reissues every identifier and effectively invalidates the whole local DB.
+
+Both look catastrophic for cursor-based sync and are a non-event here: rehash,
+diff, upload nothing because everything matches.
+
+### Handling server-side deletion
+
+The client must skip hashes the server reports as `deleted` or `banned`. Doing
+otherwise causes the **zombie photo problem** — content the user deliberately
+deleted silently resurrected by the next sync, forever. This is one of the most
+common complaints against Google Photos and Immich-style backup.
+
+This is why deletion needs tombstoned hashes server-side, and why `delete` and
+`delete + ban` are separate actions (hooks doc §10).
+
+### Re-verification
+
+Don't re-diff the whole library on every app open. Verify newly-hashed items
+immediately; re-verify the full set on a slow rolling schedule (a slice per
+day, or explicit pull-to-refresh). Hashes are already cached, so re-verification
+is pure network — cheap enough to run weekly.
+
+### Network policy
+
+- **Wi-Fi-only by default**, explicit cellular opt-in asked once at onboarding,
+ never mid-sync.
+- Detect **metered** connections, not just "is it Wi-Fi" — a tethered hotspot
+ reports as Wi-Fi and bills like cellular. Android:
+ `NET_CAPABILITY_NOT_METERED`. iOS: `NWPath.isExpensive` / `isConstrained`.
+- Prefer a **size threshold** over a binary switch: "photos on cellular, videos
+ on Wi-Fi only" is what people actually want, and matters much more with video.
+- Charging/battery constraint for the initial bulk backfill.
+- **Use the resumable chunk endpoints on cellular.** A dropped connection 90%
+ through a 200MB video should resume, not restart.
+
+Android is largely declarative — WorkManager `Constraints` covers
+metered/charging/idle and survives reboots. iOS is manual.
+
+### Platform realities
+
+- **iOS background upload is opportunistic.** `BGProcessingTask` fires when iOS
+ decides. Sync is "reliable while open, best-effort in background" — state
+ that in the UI rather than promising continuous backup.
+- **Android is reliable within limits** (WorkManager + foreground service):
+ work survives process death and reboots, but is still subject to Doze, App
+ Standby buckets, JobScheduler quotas, and aggressive OEM battery management,
+ which can defer it indefinitely. Better than iOS, not a guarantee.
+- **The iOS share extension needs native code** regardless of framework — a
+ separate app target. `expo-share-extension` handles it, but that means an
+ EAS/dev-client build, not Expo Go.
+
+Because neither platform guarantees background execution, **the UI must show
+sync state honestly** on both: a visible "N pending, last synced ", the
+reason work is currently blocked (waiting for Wi-Fi, waiting to charge, deferred
+by the OS), and permanently-failed items as actionable. No screen should imply
+continuous or completed backup that the OS has not actually performed.
+
+---
+
+## 8. Later, as plugins
+
+Each registers a background job; none touch Core.
+
+- **`gallery-dl` / `yt-dlp`** — site-aware fetching (Pixiv, Twitter…). Brings
+ original tags, artist, and post metadata. The path to video.
+- **Watch folder** — inotify on `./data/inbox`. Also the target for a
+ **Syncthing/rclone gallery-backup recipe**, which delivers cross-platform
+ camera-roll backup with *no app development at all*. Worth shipping early as
+ a hedge against the mobile app's cost.
+- **`from-url`** — server-side fetch; the shared primitive under the above.
+- **Telegram bot / email-to-upload / RSS pull** — trivial once `from-url` exists.
+
+---
+
+## 9. Build order
+
+Ordered by value-per-effort, with server prerequisites first.
+
+1. **Upload-scoped API key permission** — gates every headless client.
+2. **`findManyBySha256` + `POST /assets/exists`** — four-state; unblocks sync.
+ Depends on deletion + ban existing (hooks doc phases 2–4).
+3. **Browser extension** — the stated primary scenario; needs no new API.
+4. **Web app paste / drag-drop / needs-tagging queue** — small, independent.
+5. **PWA `share_target`** — Android mobile capture, an afternoon.
+6. **iOS Shortcut template** — documentation only.
+7. **Watch-folder plugin + Syncthing recipe** — cheap cross-platform backup.
+8. **Mobile app** — share target first, then gallery sync.
+9. **`from-url` + `gallery-dl` plugin** — video and site-aware ingest.
+
+Items 1–2 are server-side and belong with the hooks work. Items 3–6 are each
+small enough to land independently. Item 8 is a project in its own right and
+should not be started until 2 is settled — the sync contract is its foundation.
+
+---
+
+## 10. Open questions
+
+1. Extension: Chrome-only first, or Firefox parity from the start?
+2. Mobile framework: Expo/React Native vs. Capacitor vs. native. Driven mostly
+ by the iOS share-extension and background-task story.
+3. Does the extension reuse the web session cookie when same-origin, or always
+ use an API key? (API key is simpler and works cross-origin.)
+4. Should sync upload HEIC/Live Photos as-is, or transcode client-side?
+ Interacts with the transform hooks and the dual-hash problem.
+5. Per-device registration — do we want visible "devices" in account settings
+ (revocable per-device API keys), or one key per user?
diff --git a/docs/ingest-hooks.md b/docs/ingest-hooks.md
new file mode 100644
index 0000000..7f26140
--- /dev/null
+++ b/docs/ingest-hooks.md
@@ -0,0 +1,390 @@
+# Ingest Hooks — Design Plan
+
+**Status:** draft, not implemented. Revisit before building.
+
+Adds plugin-extensible hook points around the asset ingest pipeline so features
+like hash bans, EXIF stripping, and AI auto-tagging live in plugins instead of
+Core — per CLAUDE.md's Core Rule and Plugin Rule.
+
+---
+
+## 1. Goals
+
+- Let a plugin **reject** an ingest (banned hashes, quotas, file-type policy).
+- Let a plugin **rewrite the bytes** before they are stored (EXIF/metadata strip).
+- Keep the existing **fire-and-forget post-ingest** path for async work
+ (thumbnails, AI tagging, OCR).
+- Guarantee coverage of **every** upload source by construction, not by
+ discipline.
+
+### Non-goals
+
+- Content moderation. A hash ban stops *byte-identical* re-uploads only; any
+ re-encode defeats it. Perceptual/similarity matching is a separate, later,
+ background-job feature.
+- Mutating an asset's bytes *after* it has been persisted (see §7).
+
+---
+
+## 2. Why the ingest pipeline is the right seam
+
+`packages/core/src/services/asset-service.ts` has a single private `ingest()`
+function. Every upload source funnels through it:
+
+- `POST /assets` (one-shot multipart)
+- `POST /uploads/:token` (resumable chunked)
+- `plugins/shimmie-import`
+- any future watch-folder / gallery-dl / from-url plugin
+
+A hook placed inside `ingest()` therefore covers all sources **by
+construction** — a new ingest plugin cannot forget to call it. A hook placed in
+a route handler could not make that guarantee.
+
+### Current order
+
+```text
+hash (sha256 + md5, one streaming pass)
+ → dedupe on sha256
+ → sniff (Bun.Image metadata, maxPixels bomb guard)
+ → store (move fast-path or stream)
+ → insert
+ → emit asset.created
+```
+
+---
+
+## 3. Hook taxonomy
+
+Three distinct kinds. Conflating them is the main design risk.
+
+| Kind | Timing | Can veto | Can mutate bytes | Awaited | Failure default |
+|---|---|---|---|---|---|
+| `ingest.guard` | after hash, before store | yes | no | yes | **fail-closed** |
+| `ingest.transform` | before store | no | yes | yes | **fail-open** |
+| `asset.created` | after insert | no | no | no (fire-and-forget) | isolated |
+
+`asset.created` already exists (`packages/core/src/events/index.ts`) and needs
+no change. Only the two pre-ingest kinds are new.
+
+### Why guards and transforms are separate
+
+A guard answers *"may this in?"* and must be able to say no. A transform
+answers *"what exactly gets stored?"* and must not be able to say no — a broken
+EXIF stripper should never block uploads. They also want opposite failure
+defaults (§6), which alone justifies separate types.
+
+### Why the event bus can't do this
+
+`CoreEvents` is a fire-and-forget `TypedEventEmitter`; listeners are
+error-isolated and their return values are discarded. There is no way for a
+listener to refuse or alter an ingest. Guards/transforms are a **new SDK
+primitive**, not a new event.
+
+---
+
+## 4. Proposed pipeline order
+
+```text
+ hash original → originalSha256, md5 (streaming, cheap)
+→ guard(originalSha256) → may reject (403)
+→ transform(bytes) → may rewrite the blob
+→ re-hash IF transformed → sha256 (storage identity)
+→ dedupe on sha256
+→ sniff
+→ store
+→ insert
+→ emit asset.created
+```
+
+Two deliberate choices:
+
+**Guard runs on the ORIGINAL hash, before transform.** Transforms decode and
+re-encode — expensive. Rejecting a banned upload should cost one streaming hash
+and one index lookup, never a decode. It also means the ban list is expressed
+in terms of hashes a *client* can compute (§5).
+
+**Re-hash only when a transform actually changed the bytes.** With no
+transforms registered — the default — this is byte-for-byte the pipeline that
+exists today, at identical cost. Hooks are zero-overhead when unused.
+
+---
+
+## 5. The dual-hash problem
+
+**This is the subtlest interaction in the design and the main reason to write
+it down before building.**
+
+If a transform rewrites bytes, the stored `sha256` is the hash of the
+*transformed* bytes. But the planned mobile gallery sync (`POST /assets/exists`)
+has the client hash the **original** file on the device. Those hashes differ, so:
+
+1. Client hashes local photo → `abc…`
+2. Server stripped EXIF at ingest and stored `def…`
+3. `exists(abc…)` → `absent`
+4. Client re-uploads → server strips, dedupes to `def…`, returns 200
+5. Client still has no record of `abc…` → **re-uploads forever, every sync**
+
+### Resolution: store both hashes
+
+| column | meaning | unique | indexed |
+|---|---|---|---|
+| `sha256` | transformed bytes; storage key + dedupe identity | yes | yes (exists) |
+| `originalSha256` | bytes as received; what clients can compute | no | yes (new) |
+
+- `/assets/exists` queries **`originalSha256`**.
+- Ban list matches on **`originalSha256`** (a client/mod bans what they saw).
+- Storage key and dedupe continue to use **`sha256`**.
+- With no transform registered the two are equal, so this is inert by default.
+
+Cost: one nullable-or-equal text column + one btree index. Cheap now,
+retrofit-hostile later (originals are unrecoverable once transformed).
+
+**Open question:** should `originalSha256` be a 1:N side table
+(`asset_source_hashes`) instead? Several different originals (same image,
+differing EXIF) can transform to one stored asset — a column only records the
+first one, so subsequent variants would re-upload once each before deduping.
+A side table records them all and makes sync exact. Leaning side table; decide
+before implementing.
+
+### Second interaction: the move fast-path
+
+`ingest()` has a zero-copy path — `storage.ingestLocalFile(localPath, key)` —
+that *moves* a staged file into place. A transform produces new bytes, so it
+**invalidates that path**; a transformed ingest must write the new blob. The
+implementation must skip the move whenever any transform mutated the source,
+and keep it whenever none did.
+
+---
+
+## 6. Failure semantics
+
+Different defaults per kind, deliberately:
+
+**Guards fail closed.** A guard that throws or times out → the ingest is
+rejected. A security feature that fails open is not a security feature. The
+mitigation for a broken guard is operator control: per-plugin
+enable/disable, so a bad guard can be switched off without a redeploy.
+
+**Transforms fail open.** A transform that throws or times out → log loudly,
+skip that transform, ingest the original bytes. Failing to strip EXIF is
+cosmetic; refusing the upload is not.
+
+**Transform output is validated before it is trusted.** `apply()` returns a
+`Blob` from plugin code; the pipeline must confirm it is non-empty and sniffs as
+a supported format *before* it becomes the stored bytes. On validation failure,
+fall back to the last known-good transformed blob, or the original bytes if no
+transform has succeeded yet — same loud-log-and-continue posture as a throw.
+Without this, a transform that silently returns garbage turns a fail-open hook
+into a corrupted-asset hook.
+
+**Both are bounded by a timeout** (proposed: 5s guard, 30s transform,
+configurable). Timing out must *cancel the hook's work*, not merely stop waiting
+for it: hooks receive an `AbortSignal` in `IngestContext` and are contractually
+required to honour it. A plugin that ignores the signal cannot be forcibly
+killed (same process, no isolation), so the pipeline abandons its result,
+proceeds under the failure default for its kind, and logs the offending plugin
+id — an uncooperative hook leaks a task, which is a bug to surface, not to hide.
+
+**Ordering is deterministic:** sort by the hook's numeric `order`, **defaulting
+to `0`** when omitted, then by plugin id, then by the hook's index within that
+plugin's registration array. The last key matters because one plugin may
+register several hooks with equal `order`; without it their relative order
+depends on registration timing and transform output could differ across
+restarts.
+
+**Neither kind runs on a dedupe hit** — but note *when* the dedupe hit is known.
+Per §4 the guard runs before dedupe (deliberately: a banned original must be
+rejected even if identical bytes are already stored, or the ban is trivially
+bypassed by uploading a copy). So "does not run on a dedupe hit" applies to
+**transforms only**; the guard has already run by then. What dedupe skips is the
+transform, sniff, store, and insert work. §4's ordering is authoritative.
+
+---
+
+## 7. Rule: nothing mutates a persisted asset
+
+Post-ingest work (thumbnails, AI enhancement, format conversion) MUST NOT
+rewrite the stored original. The storage key is content-addressed
+(`assets///.`), so changing bytes after insert would
+invalidate the key, the dedupe identity, and any ban entry simultaneously.
+
+Derivatives belong in **plugin-namespaced storage** — which
+`PluginContext.storage` already provides (keys transparently prefixed with
+`plugins//`). This is exactly what `plugins/thumbnailer` does today; the
+pattern generalizes to enhancement and transcoding.
+
+"Post-enhancement" is therefore a *derivative producer*, not a mutating hook,
+and needs no new primitive beyond `asset.created`.
+
+---
+
+## 8. SDK changes
+
+`packages/plugin-sdk/src/index.ts`:
+
+- Add `"ingest-guards"` and `"ingest-transforms"` to `SDK_CAPABILITIES`.
+- Bump `PLUGIN_SDK_VERSION` → `0.3.0` (additive, but capability vocabulary is
+ public contract).
+- Extend `PluginRegistration` with optional `ingestGuards` / `ingestTransforms`.
+
+Sketch (names provisional):
+
+```ts
+/** What a guard/transform sees. Bytes are lazy — a guard need not read them. */
+export interface IngestContext {
+ readonly originalSha256: string;
+ readonly md5: string;
+ readonly sizeBytes: number;
+ readonly declaredMimeType: string | null;
+ readonly uploaderId: number | null;
+ readonly source: string | null;
+ readonly blob: Blob;
+}
+
+export type IngestVerdict = { allow: true } | { allow: false; reason: string };
+
+export interface IngestGuard {
+ order?: number;
+ check(ctx: IngestContext): Promise | IngestVerdict;
+}
+
+export interface IngestTransform {
+ order?: number;
+ /** Return null to pass through unchanged (the common case). */
+ apply(ctx: IngestContext): Promise | Blob | null;
+}
+```
+
+Core wiring: `createAssetService(repository, storage, events, hooks?)` gains an
+optional hook registry, populated by the plugin loader before the first ingest.
+A denied verdict surfaces as `403` carrying `reason`.
+
+**Note:** the sniffed MIME/dimensions are not available to guards, because
+sniffing happens after the store decision. A guard needing real format (vs.
+client-declared) must sniff the blob itself, or we accept a second sniff pass.
+Open question.
+
+---
+
+## 9. First consumers
+
+| Plugin | Kind | Notes |
+|---|---|---|
+| `banned-hashes` | guard | own table, admin page, ban/unban routes |
+| `exif-strip` | transform | drives the dual-hash design in §5 |
+| `auto-tag` (AI) | `asset.created` | existing event; no new primitive |
+| `thumbnailer` | `asset.created` | already shipped; unchanged |
+
+### `banned-hashes` sketch
+
+Own table (plugin-owned, per the SDK's tables capability):
+
+```text
+banned_hashes(sha256, md5, reason, banned_by, banned_at)
+```
+
+- Guard: reject when `originalSha256` is listed → `403`.
+- Retains the hash after the binary is purged — the whole mechanism depends on
+ the row outliving the file.
+- Store `md5` alongside for booru-ecosystem interop (imported ban lists from
+ other boorus are md5-keyed).
+- Admin page + moderator-gated ban/unban routes.
+
+---
+
+## 10. Interaction with asset deletion and mobile sync
+
+Assets currently have **no delete path** (no repository `delete`, no
+`deletedAt`). When deletion is designed it must account for this plan:
+
+**Delete and ban are separate actions.** Most deletes are "wrong crop,
+re-uploading the good one" — auto-banning on delete makes that unfixable.
+Ban is a deliberate second step (matching Danbooru's mod flow).
+
+That gives `POST /assets/exists` four states per hash:
+
+| state | meaning | sync client | upload |
+|---|---|---|---|
+| `present` | already stored | skip | 200 dedupe |
+| `absent` | never seen | upload | 201 |
+| `deleted` | removed, not banned | skip | allowed |
+| `banned` | guard-rejected | skip | 403 |
+
+Skipping `deleted` is what prevents the zombie-photo problem (deleted content
+resurrected by the next sync).
+
+**The four states overlap, so precedence must be explicit.** A hash can be
+present *and* banned (banned after it was stored, before it was deleted), or
+deleted *and* banned (the `delete + ban` action). One state is returned per
+hash, resolved highest-first:
+
+```text
+banned > deleted > present > absent
+```
+
+- `banned` wins over everything: an upload of it is `403` even if bytes are
+ still stored, and the client must never re-upload. This follows from §4 —
+ the guard runs before dedupe, so a banned hash cannot be laundered through a
+ dedupe hit.
+- `deleted` outranks `present` so a soft-deleted row still reads as `deleted`
+ while its bytes await GC.
+- Uploads follow the same precedence: `banned` → `403`; otherwise `present` →
+ `200` dedupe; `deleted` or `absent` → `201`.
+
+Consequence: `banned` is the only state that changes upload *behaviour*.
+`deleted` only changes *client* behaviour (don't re-upload), which is why
+banning must be a deliberate action rather than implied by deletion.
+
+**Auth and scoping:** `/assets/exists` is a batch hash oracle — it reveals
+whether a specific image is in the library. Require an upload-scoped API key and
+rate-limit it. Beyond that, **the visibility of `deleted` and `banned` must be
+decided before the endpoint ships**: both leak moderation state. Proposed
+default — `present`/`absent` answer for the whole library (dedupe is global, so
+this is already observable via upload response codes), while `deleted` collapses
+to `absent` unless the caller owns the asset or can moderate. `banned` is
+reported to everyone, because a client that doesn't know a hash is banned will
+retry it forever. Confirm before implementing (§11).
+
+---
+
+## 11. Open questions
+
+1. `originalSha256` column vs. `asset_source_hashes` side table (§5).
+2. Do guards get sniffed format, or sniff themselves (§8)?
+3. Guard fail-closed default — acceptable, given per-plugin disable?
+4. Are transform hooks operator-orderable in settings, or fixed at code order?
+5. Should `banned-hashes` be seedable from an external list at boot?
+6. Owner-scoping of `deleted`/`banned` in `/assets/exists` (§10) — confirm the
+ proposed default before Phase 4.
+7. Once (1) is settled, does `banned_hashes.sha256` mean the original or the
+ stored hash? §5 says ban matches `originalSha256`, so the column is
+ arguably misnamed — rename with (1), not before.
+
+---
+
+## 12. Phased implementation
+
+Sized to land as coherent PRs rather than many small ones.
+
+**Phase 1 — hook primitive (Core + SDK)**
+Hook registry, pipeline reorder, guard/transform types, capabilities, timeouts,
+failure semantics, loader wiring. No behaviour change with zero hooks
+registered. Tests: ordering, veto, transform rewrite, timeout, fail-open vs
+fail-closed, move fast-path invalidation.
+
+**Phase 2 — `banned-hashes` plugin**
+Table, guard, admin page, ban/unban routes. First real consumer; validates the
+primitive.
+
+**Phase 3 — asset deletion + `delete + ban`**
+Soft delete, tombstones, the four-state model.
+
+**Phase 4 — `findManyBySha256` + `POST /assets/exists`**
+Batch existence check returning the four states. Unblocks mobile sync.
+
+**Phase 5 — `exif-strip` plugin**
+First transform. Forces the dual-hash work in §5 to be real; deliberately last
+so the sync contract is settled before hashes can diverge.
+
+Ingest clients (browser extension, PWA share target, mobile app) are tracked
+separately — they depend only on Phase 4.
diff --git a/packages/core/src/services/auth-service.ts b/packages/core/src/services/auth-service.ts
index 79edcf3..992de7b 100644
--- a/packages/core/src/services/auth-service.ts
+++ b/packages/core/src/services/auth-service.ts
@@ -74,6 +74,12 @@ export interface AuthService {
listApiKeys(userId: number): Promise;
/** Revoke one of the user's API keys; true if a key was removed. */
revokeApiKey(userId: number, id: number): Promise;
+ /**
+ * Look up a user by username (case-insensitive), or null. Returns the
+ * public projection (never the password hash) — used e.g. by the importer to
+ * resolve/validate a target user before attributing imported posts to them.
+ */
+ findByUsername(username: string): Promise;
}
/** Configuration for {@link createAuthService}. */
@@ -222,5 +228,13 @@ export function createAuthService(
revokeApiKey(userId, id) {
return apiKeys.deleteByIdForUser(id, userId);
},
+
+ async findByUsername(username) {
+ const user = await users.findByUsername(normalizeUsername(username));
+ if (!user) return null;
+ // Strip the password hash — callers only ever need the public projection.
+ const { passwordHash: _passwordHash, ...publicUser } = user;
+ return publicUser;
+ },
};
}
diff --git a/packages/core/test/auth-service.test.ts b/packages/core/test/auth-service.test.ts
index 4aac8c1..b72d4a7 100644
--- a/packages/core/test/auth-service.test.ts
+++ b/packages/core/test/auth-service.test.ts
@@ -164,6 +164,24 @@ describe("createAuthService.register", () => {
});
});
+describe("createAuthService.findByUsername", () => {
+ it("resolves a user case-insensitively, without the password hash", async () => {
+ const { service } = makeService();
+ const { user } = await service.register({ username: "Alice", password: "supersecret" });
+
+ const found = await service.findByUsername("ALICE");
+ expect(found?.id).toBe(user.id);
+ expect(found?.username).toBe("alice");
+ // PublicUser projection — the hash must never be exposed.
+ expect(found !== null && "passwordHash" in found).toBe(false);
+ });
+
+ it("returns null for an unknown user", async () => {
+ const { service } = makeService();
+ expect(await service.findByUsername("nobody")).toBeNull();
+ });
+});
+
describe("createAuthService.login", () => {
it("rejects an unknown user and a wrong password, accepts correct (case-insensitive) creds", async () => {
const { service } = makeService();
diff --git a/plugins/shimmie-import/drizzle/0001_whole_shotgun.sql b/plugins/shimmie-import/drizzle/0001_whole_shotgun.sql
new file mode 100644
index 0000000..815f869
--- /dev/null
+++ b/plugins/shimmie-import/drizzle/0001_whole_shotgun.sql
@@ -0,0 +1,7 @@
+-- Add run_id safely on a non-empty table: Postgres rejects ADD COLUMN NOT NULL
+-- without a DEFAULT when rows exist. Add with a transient DEFAULT (backfilling any
+-- pre-existing rows with 0 — a sentinel that matches no real run), then drop it so
+-- future inserts must supply run_id (the plugin always does). End state matches the
+-- Drizzle snapshot: NOT NULL, no default.
+ALTER TABLE "shimmie_import_items" ADD COLUMN "run_id" integer DEFAULT 0 NOT NULL;--> statement-breakpoint
+ALTER TABLE "shimmie_import_items" ALTER COLUMN "run_id" DROP DEFAULT;
diff --git a/plugins/shimmie-import/drizzle/0002_next_steel_serpent.sql b/plugins/shimmie-import/drizzle/0002_next_steel_serpent.sql
new file mode 100644
index 0000000..039dccc
--- /dev/null
+++ b/plugins/shimmie-import/drizzle/0002_next_steel_serpent.sql
@@ -0,0 +1 @@
+CREATE INDEX "shimmie_import_items_run_status_idx" ON "shimmie_import_items" USING btree ("run_id","status");
\ No newline at end of file
diff --git a/plugins/shimmie-import/drizzle/meta/0001_snapshot.json b/plugins/shimmie-import/drizzle/meta/0001_snapshot.json
new file mode 100644
index 0000000..45b837b
--- /dev/null
+++ b/plugins/shimmie-import/drizzle/meta/0001_snapshot.json
@@ -0,0 +1,207 @@
+{
+ "id": "ff7d51c0-8aa8-4f40-8165-004255d12670",
+ "prevId": "7ce6e4cf-5f72-43ca-9efb-74ccfdfea9bb",
+ "version": "7",
+ "dialect": "postgresql",
+ "tables": {
+ "public.shimmie_import_items": {
+ "name": "shimmie_import_items",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "source_instance": {
+ "name": "source_instance",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "source_post_id": {
+ "name": "source_post_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "run_id": {
+ "name": "run_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "asset_id": {
+ "name": "asset_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "error": {
+ "name": "error",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "shimmie_import_items_source_instance_source_post_id_unique": {
+ "name": "shimmie_import_items_source_instance_source_post_id_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "source_instance",
+ "source_post_id"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.shimmie_import_runs": {
+ "name": "shimmie_import_runs",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "source_instance": {
+ "name": "source_instance",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "user_filter": {
+ "name": "user_filter",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "source_timezone": {
+ "name": "source_timezone",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'UTC'"
+ },
+ "target_user_id": {
+ "name": "target_user_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "max_id": {
+ "name": "max_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "cursor": {
+ "name": "cursor",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "imported": {
+ "name": "imported",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "failed": {
+ "name": "failed",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "skipped": {
+ "name": "skipped",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "status": {
+ "name": "status",
+ "type": "shimmie_import_run_status",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'running'"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ }
+ },
+ "enums": {
+ "public.shimmie_import_run_status": {
+ "name": "shimmie_import_run_status",
+ "schema": "public",
+ "values": [
+ "running",
+ "done",
+ "canceled"
+ ]
+ }
+ },
+ "schemas": {},
+ "sequences": {},
+ "roles": {},
+ "policies": {},
+ "views": {},
+ "_meta": {
+ "columns": {},
+ "schemas": {},
+ "tables": {}
+ }
+}
\ No newline at end of file
diff --git a/plugins/shimmie-import/drizzle/meta/0002_snapshot.json b/plugins/shimmie-import/drizzle/meta/0002_snapshot.json
new file mode 100644
index 0000000..a76cf87
--- /dev/null
+++ b/plugins/shimmie-import/drizzle/meta/0002_snapshot.json
@@ -0,0 +1,229 @@
+{
+ "id": "7f46c7b5-90f8-41fd-96cc-81cefeb523c6",
+ "prevId": "ff7d51c0-8aa8-4f40-8165-004255d12670",
+ "version": "7",
+ "dialect": "postgresql",
+ "tables": {
+ "public.shimmie_import_items": {
+ "name": "shimmie_import_items",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "source_instance": {
+ "name": "source_instance",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "source_post_id": {
+ "name": "source_post_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "run_id": {
+ "name": "run_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "asset_id": {
+ "name": "asset_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "error": {
+ "name": "error",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {
+ "shimmie_import_items_run_status_idx": {
+ "name": "shimmie_import_items_run_status_idx",
+ "columns": [
+ {
+ "expression": "run_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "status",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "shimmie_import_items_source_instance_source_post_id_unique": {
+ "name": "shimmie_import_items_source_instance_source_post_id_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "source_instance",
+ "source_post_id"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.shimmie_import_runs": {
+ "name": "shimmie_import_runs",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "source_instance": {
+ "name": "source_instance",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "user_filter": {
+ "name": "user_filter",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "source_timezone": {
+ "name": "source_timezone",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'UTC'"
+ },
+ "target_user_id": {
+ "name": "target_user_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "max_id": {
+ "name": "max_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "cursor": {
+ "name": "cursor",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "imported": {
+ "name": "imported",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "failed": {
+ "name": "failed",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "skipped": {
+ "name": "skipped",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "status": {
+ "name": "status",
+ "type": "shimmie_import_run_status",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'running'"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ }
+ },
+ "enums": {
+ "public.shimmie_import_run_status": {
+ "name": "shimmie_import_run_status",
+ "schema": "public",
+ "values": [
+ "running",
+ "done",
+ "canceled"
+ ]
+ }
+ },
+ "schemas": {},
+ "sequences": {},
+ "roles": {},
+ "policies": {},
+ "views": {},
+ "_meta": {
+ "columns": {},
+ "schemas": {},
+ "tables": {}
+ }
+}
\ No newline at end of file
diff --git a/plugins/shimmie-import/drizzle/meta/_journal.json b/plugins/shimmie-import/drizzle/meta/_journal.json
index aebd0ed..2a7cf4b 100644
--- a/plugins/shimmie-import/drizzle/meta/_journal.json
+++ b/plugins/shimmie-import/drizzle/meta/_journal.json
@@ -8,6 +8,20 @@
"when": 1785626693919,
"tag": "0000_lying_tomas",
"breakpoints": true
+ },
+ {
+ "idx": 1,
+ "version": "7",
+ "when": 1785634085457,
+ "tag": "0001_whole_shotgun",
+ "breakpoints": true
+ },
+ {
+ "idx": 2,
+ "version": "7",
+ "when": 1785636312922,
+ "tag": "0002_next_steel_serpent",
+ "breakpoints": true
}
]
}
\ No newline at end of file
diff --git a/plugins/shimmie-import/src/import.ts b/plugins/shimmie-import/src/import.ts
index d6e8f53..9a93abc 100644
--- a/plugins/shimmie-import/src/import.ts
+++ b/plugins/shimmie-import/src/import.ts
@@ -1,9 +1,9 @@
-import { and, eq, ne } from "drizzle-orm";
+import { and, eq, ne, sql } from "drizzle-orm";
import type { PluginContext } from "@bunbooru/plugin-sdk";
import { importItems, importRuns } from "./schema";
-import type { SourceAdapter } from "./source-adapter";
+import type { SourceAdapter, SourcePost } from "./source-adapter";
/** Posts ATTEMPTED (imported + failed) per `step` call. */
const IMPORT_BATCH = 25;
@@ -48,20 +48,77 @@ async function upsertItem(
ctx: PluginContext,
sourceInstance: string,
sourcePostId: number,
- fields: { assetId: number | null; status: "complete" | "failed"; error: string | null },
+ runId: number,
+ // Discriminated union: a `complete` row always carries the asset it produced,
+ // a `failed` row never does — so the ledger can't record a success with no
+ // asset (or a failure that still points at one).
+ fields:
+ | { assetId: number; status: "complete"; error: null }
+ | { assetId: null; status: "failed"; error: string },
): Promise {
await ctx.db
.insert(importItems)
- .values({ sourceInstance, sourcePostId, ...fields })
+ .values({ sourceInstance, sourcePostId, runId, ...fields })
.onConflictDoUpdate({
target: [importItems.sourceInstance, importItems.sourcePostId],
- set: { ...fields, updatedAt: new Date() },
+ // Record the run that last touched it (scopes retry-failed).
+ set: { ...fields, runId, updatedAt: new Date() },
// `complete` is terminal: never let a later (e.g. concurrent) `failed`
// overwrite a successful import and clear its assetId.
setWhere: ne(importItems.status, "complete"),
});
}
+/**
+ * Ingest one already-fetched post: download bytes → `create` (preserving
+ * rating/source/date, attributing to the target user) → `setAssetTags` → record
+ * the outcome in the ledger. Never throws — a failure records a `failed` row and
+ * returns "failed". Shared by {@link stepRun} and {@link retryFailed}.
+ */
+async function ingestPost(
+ ctx: PluginContext,
+ adapter: SourceAdapter,
+ post: SourcePost,
+ runId: number,
+ targetUserId: number,
+): Promise<"complete" | "failed"> {
+ // Track the asset once created so a LATER failure (e.g. setAssetTags) is logged
+ // WITH the asset id — the created asset isn't lost even though the ledger's
+ // `failed` row can't carry an assetId. A retry re-runs (sha256 dedupe returns
+ // the same asset) and re-applies the tags, recovering it.
+ let createdAssetId: number | undefined;
+ try {
+ const bytes = await adapter.fetchBytes(post);
+ const { asset } = await ctx.services.assets.create({
+ bytes,
+ rating: post.rating,
+ source: post.postUrl,
+ uploaderId: targetUserId,
+ createdAt: post.postedAt,
+ });
+ createdAssetId = asset.id;
+ if (post.tags.length > 0) await ctx.services.tags.setAssetTags(asset.id, post.tags);
+ await upsertItem(ctx, adapter.sourceInstance, post.sourcePostId, runId, {
+ assetId: asset.id,
+ status: "complete",
+ error: null,
+ });
+ return "complete";
+ } catch (error) {
+ await upsertItem(ctx, adapter.sourceInstance, post.sourcePostId, runId, {
+ assetId: null,
+ status: "failed",
+ error: errorMessage(error),
+ });
+ ctx.log.warn("import_post_failed", {
+ sourcePostId: post.sourcePostId,
+ createdAssetId,
+ error: errorMessage(error),
+ });
+ return "failed";
+ }
+}
+
/** Whether this source post was already imported successfully (skip on re-run). */
async function alreadyComplete(
ctx: PluginContext,
@@ -118,12 +175,12 @@ export async function stepRun(
scanned += 1;
const sourcePostId = cursor;
- let post;
+ let post: Awaited>;
try {
post = await adapter.fetchPost(sourcePostId);
} catch (error) {
// A fetch/GraphQL error (not a deletion) is recorded + retryable on a new run.
- await upsertItem(ctx, adapter.sourceInstance, sourcePostId, {
+ await upsertItem(ctx, adapter.sourceInstance, sourcePostId, run.id, {
assetId: null,
status: "failed",
error: errorMessage(error),
@@ -145,30 +202,10 @@ export async function stepRun(
continue;
}
- try {
- const bytes = await adapter.fetchBytes(post);
- const { asset } = await ctx.services.assets.create({
- bytes,
- rating: post.rating,
- source: post.postUrl,
- uploaderId: run.targetUserId,
- createdAt: post.postedAt,
- });
- if (post.tags.length > 0) await ctx.services.tags.setAssetTags(asset.id, post.tags);
- await upsertItem(ctx, adapter.sourceInstance, sourcePostId, {
- assetId: asset.id,
- status: "complete",
- error: null,
- });
+ if ((await ingestPost(ctx, adapter, post, run.id, run.targetUserId)) === "complete") {
imported += 1;
- } catch (error) {
- await upsertItem(ctx, adapter.sourceInstance, sourcePostId, {
- assetId: null,
- status: "failed",
- error: errorMessage(error),
- });
+ } else {
failed += 1;
- ctx.log.warn("import_post_failed", { sourcePostId, error: errorMessage(error) });
}
}
@@ -179,10 +216,13 @@ export async function stepRun(
skipped: run.skipped + skipped,
};
// Optimistic concurrency: only commit the new absolute totals if no other
- // step advanced the cursor since we read it (WHERE cursor = the value we read).
- // If a concurrent step won, we drop this counter write — our per-post ledger
- // rows still stand, and Core's sha256 dedupe prevents duplicate assets.
- await ctx.db
+ // step advanced the cursor since we read it (WHERE cursor = the value we read)
+ // AND the run is still `running`. The status guard matters because a step is
+ // long (network-bound) and `cancel` can land mid-flight: without it, this
+ // write would resurrect a canceled run back to `running`/`done`.
+ // If either guard fails, we drop this counter write — our per-post ledger rows
+ // still stand, and Core's sha256 dedupe prevents duplicate assets.
+ const committed = await ctx.db
.update(importRuns)
.set({
cursor,
@@ -192,7 +232,147 @@ export async function stepRun(
status: done ? "done" : "running",
updatedAt: new Date(),
})
- .where(and(eq(importRuns.id, runId), eq(importRuns.cursor, run.cursor)));
+ .where(
+ and(
+ eq(importRuns.id, runId),
+ eq(importRuns.cursor, run.cursor),
+ eq(importRuns.status, "running"),
+ ),
+ )
+ .returning({ id: importRuns.id });
+
+ if (committed.length === 0) {
+ // Our write lost (cancel, or a concurrent step). The per-step counts below
+ // are still true — that work really happened — but cursor/done/totals must
+ // report the PERSISTED run, not our dropped local view, or the client would
+ // act on a state the database never accepted.
+ const currentRows = await ctx.db
+ .select()
+ .from(importRuns)
+ .where(eq(importRuns.id, runId))
+ .limit(1);
+ const current = currentRows[0];
+ if (current) {
+ return {
+ imported,
+ failed,
+ skipped,
+ cursor: current.cursor,
+ maxId: current.maxId,
+ done: current.status !== "running",
+ totals: {
+ imported: current.imported,
+ failed: current.failed,
+ skipped: current.skipped,
+ },
+ };
+ }
+ }
return { imported, failed, skipped, cursor, maxId: run.maxId, done, totals };
}
+
+/** Failed posts re-attempted per `retry-failed` call. */
+const RETRY_BATCH = 25;
+
+/** Outcome of one {@link retryFailed} call. */
+export interface RetryResult {
+ /** Failed items attempted this call. */
+ retried: number;
+ /** Now imported. */
+ recovered: number;
+ /** Still failing after the retry. */
+ stillFailed: number;
+ /** Failed items remaining for THIS run after this call. */
+ remainingFailed: number;
+}
+
+/** Count the `failed` ledger rows owned by a run. */
+async function countFailed(ctx: PluginContext, runId: number): Promise {
+ const rows = await ctx.db
+ .select({ n: sql`count(*)::int` })
+ .from(importItems)
+ .where(and(eq(importItems.runId, runId), eq(importItems.status, "failed")));
+ return rows[0]?.n ?? 0;
+}
+
+/**
+ * Re-attempt up to {@link RETRY_BATCH} previously-`failed` posts of ONE run
+ * (e.g. after a transient network blip), without re-scanning the whole id space.
+ * Scoped by `runId` (not the whole source) so it can't re-attribute another run's
+ * posts to this run's target user. A source post that has since been deleted has
+ * its stale `failed` row removed. Recovered items move from the run's `failed`
+ * counter to `imported`. The client loops until this run's failures reach 0 OR a
+ * batch recovers nothing (permanent failures).
+ */
+export async function retryFailed(
+ ctx: PluginContext,
+ adapter: SourceAdapter,
+ runId: number,
+): Promise {
+ const runRows = await ctx.db.select().from(importRuns).where(eq(importRuns.id, runId)).limit(1);
+ const run = runRows[0];
+ if (!run) throw new Error("Import run not found");
+ // A canceled run is terminal — don't resurrect it by retrying/updating counters.
+ if (run.status === "canceled") throw new Error("Cannot retry a canceled import run");
+
+ const failedRows = await ctx.db
+ .select({ sourcePostId: importItems.sourcePostId })
+ .from(importItems)
+ .where(and(eq(importItems.runId, runId), eq(importItems.status, "failed")))
+ .orderBy(importItems.sourcePostId)
+ .limit(RETRY_BATCH);
+
+ let recovered = 0;
+ let stillFailed = 0;
+ let deleted = 0;
+
+ for (const { sourcePostId } of failedRows) {
+ let post;
+ try {
+ post = await adapter.fetchPost(sourcePostId);
+ } catch (error) {
+ await upsertItem(ctx, adapter.sourceInstance, sourcePostId, runId, {
+ assetId: null,
+ status: "failed",
+ error: errorMessage(error),
+ });
+ stillFailed += 1;
+ continue;
+ }
+ if (!post) {
+ // Source post is gone now — drop the stale failed row (nothing to recover).
+ await ctx.db
+ .delete(importItems)
+ .where(
+ and(eq(importItems.sourceInstance, adapter.sourceInstance), eq(importItems.sourcePostId, sourcePostId)),
+ );
+ deleted += 1;
+ continue;
+ }
+ if ((await ingestPost(ctx, adapter, post, runId, run.targetUserId)) === "complete") recovered += 1;
+ else stillFailed += 1;
+ }
+
+ // A recovered failure becomes an import; a deleted stale failure just disappears.
+ // Both reduce the run's `failed` count. Relative SQL increments (not absolute
+ // values from the stale snapshot) so a concurrent step's write isn't clobbered,
+ // and the status guard keeps a concurrently-canceled run terminal.
+ if (recovered > 0 || deleted > 0) {
+ await ctx.db
+ .update(importRuns)
+ .set({
+ imported: sql`${importRuns.imported} + ${recovered}`,
+ failed: sql`greatest(${importRuns.failed} - ${recovered + deleted}, 0)`,
+ updatedAt: new Date(),
+ })
+ .where(and(eq(importRuns.id, runId), ne(importRuns.status, "canceled")));
+ }
+
+ return {
+ retried: failedRows.length,
+ recovered,
+ stillFailed,
+ remainingFailed: await countFailed(ctx, runId),
+ };
+}
diff --git a/plugins/shimmie-import/src/index.ts b/plugins/shimmie-import/src/index.ts
index de62119..7728de2 100644
--- a/plugins/shimmie-import/src/index.ts
+++ b/plugins/shimmie-import/src/index.ts
@@ -13,7 +13,7 @@ import {
type User,
} from "@bunbooru/plugin-sdk";
-import { stepRun } from "./import";
+import { retryFailed, stepRun } from "./import";
import { importRuns } from "./schema";
import { ShimmieAdapter } from "./shimmie-adapter";
@@ -86,7 +86,18 @@ export function buildImportRoutes(ctx: PluginContext) {
const adapter = makeAdapter(body.baseUrl, body.apiKey, timezone);
const { maxId } = await adapter.preflight();
const userFilter = body.users === "*" ? "*" : body.users.join(",");
- const targetUserId = body.targetUserId ?? admin.id;
+ // Resolve the target bunbooru user by username (default: the admin
+ // running the import). Validate up front so a typo fails here, not on
+ // every post with an invalid uploaderId FK during stepping.
+ let targetUserId = admin.id;
+ const targetUsername = body.targetUsername?.trim();
+ if (targetUsername) {
+ const target = await ctx.services.auth.findByUsername(targetUsername);
+ if (!target) {
+ return { ok: false as const, error: `No bunbooru user named "${targetUsername}"` };
+ }
+ targetUserId = target.id;
+ }
const inserted = await ctx.db
.insert(importRuns)
.values({ sourceInstance: adapter.sourceInstance, userFilter, sourceTimezone: timezone, targetUserId, maxId })
@@ -99,20 +110,25 @@ export function buildImportRoutes(ctx: PluginContext) {
}
},
{
- body: t.Object({
- baseUrl: t.String({ minLength: 1, maxLength: 2048 }),
- apiKey: t.String({ minLength: 1, maxLength: 500 }),
- // `*` = all users, else an explicit list of shimmie usernames.
- users: t.Union([
- t.Literal("*"),
- // At least one username (an empty list would match nobody and import zero).
- t.Array(t.String({ maxLength: 100 }), { minItems: 1, maxItems: 100 }),
- ]),
- // Target bunbooru user id to attribute posts to; defaults to the admin.
- targetUserId: t.Optional(t.Integer({ minimum: 1 })),
- // IANA timezone of the source's naive timestamps (default UTC).
- sourceTimezone: t.Optional(t.String({ maxLength: 64 })),
- }),
+ body: t.Object(
+ {
+ baseUrl: t.String({ minLength: 1, maxLength: 2048 }),
+ apiKey: t.String({ minLength: 1, maxLength: 500 }),
+ // `*` = all users, else an explicit list of shimmie usernames.
+ users: t.Union([
+ t.Literal("*"),
+ // At least one username (an empty list would match nobody and import zero).
+ t.Array(t.String({ maxLength: 100 }), { minItems: 1, maxItems: 100 }),
+ ]),
+ // Target bunbooru username to attribute posts to; defaults to the admin.
+ targetUsername: t.Optional(t.String({ maxLength: 100 })),
+ // IANA timezone of the source's naive timestamps (default UTC).
+ sourceTimezone: t.Optional(t.String({ maxLength: 64 })),
+ },
+ // Reject unknown fields (e.g. a stale `targetUserId` from an old client)
+ // rather than silently ignoring them and defaulting the target to the admin.
+ { additionalProperties: false },
+ ),
},
)
// Process one bounded batch of a run. The client loops this until `done`.
@@ -143,6 +159,35 @@ export function buildImportRoutes(ctx: PluginContext) {
body: t.Object({ apiKey: t.String({ minLength: 1, maxLength: 500 }) }),
},
)
+ // Re-attempt a batch of this source's previously-failed posts. The client
+ // loops until `remainingFailed` is 0.
+ .post(
+ "/runs/:id/retry-failed",
+ async ({ params, body, request }) => {
+ await requireAdmin(ctx, request);
+ try {
+ const runRows = await ctx.db
+ .select({
+ sourceInstance: importRuns.sourceInstance,
+ sourceTimezone: importRuns.sourceTimezone,
+ })
+ .from(importRuns)
+ .where(eq(importRuns.id, params.id))
+ .limit(1);
+ const run = runRows[0];
+ if (!run) return { ok: false as const, error: "Import run not found" };
+ const adapter = makeAdapter(run.sourceInstance, body.apiKey, run.sourceTimezone);
+ const result = await retryFailed(ctx, adapter, params.id);
+ return { ok: true as const, ...result };
+ } catch (error) {
+ return { ok: false as const, error: errorMessage(error) };
+ }
+ },
+ {
+ params: t.Object({ id: t.Numeric({ minimum: 1, multipleOf: 1 }) }),
+ body: t.Object({ apiKey: t.String({ minLength: 1, maxLength: 500 }) }),
+ },
+ )
// A run's progress (admin-only).
.get(
"/runs/:id",
diff --git a/plugins/shimmie-import/src/schema.ts b/plugins/shimmie-import/src/schema.ts
index 511db9c..4a16f90 100644
--- a/plugins/shimmie-import/src/schema.ts
+++ b/plugins/shimmie-import/src/schema.ts
@@ -1,4 +1,4 @@
-import { integer, pgEnum, pgTable, serial, text, timestamp, unique } from "drizzle-orm/pg-core";
+import { index, integer, pgEnum, pgTable, serial, text, timestamp, unique } from "drizzle-orm/pg-core";
/** The lifecycle of an import run — constrained at the DB level (no stray values). */
export const importRunStatus = pgEnum("shimmie_import_run_status", ["running", "done", "canceled"]);
@@ -43,6 +43,9 @@ export const importItems = pgTable(
id: serial("id").primaryKey(),
sourceInstance: text("source_instance").notNull(),
sourcePostId: integer("source_post_id").notNull(),
+ /** The run that last processed this post — scopes retry-failed to a run so it
+ * can't re-attribute another run's posts to the wrong target user. */
+ runId: integer("run_id").notNull(),
/** The bunbooru asset created (or deduped onto); null on failure. */
assetId: integer("asset_id"),
/** `complete` | `failed`. */
@@ -51,5 +54,9 @@ export const importItems = pgTable(
createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(),
updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow().notNull(),
},
- (table) => [unique().on(table.sourceInstance, table.sourcePostId)],
+ (table) => [
+ unique().on(table.sourceInstance, table.sourcePostId),
+ // retry-failed selects WHERE run_id = ? AND status = 'failed'; index it.
+ index("shimmie_import_items_run_status_idx").on(table.runId, table.status),
+ ],
);