diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index a40acfb..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -version: 2.1 -jobs: - test: - docker: - - image: girder/girder_test:latest - - image: circleci/mongo:4.0-ram - command: ["mongod", "--storageEngine", "ephemeralForTest", "--dbpath", "/dev/shm/mongo"] - - steps: - - checkout - - run: - name: Run server tests - command: tox - - run: - name: Run web tests - command: | - npm install - npm run lint - working_directory: girder_volview/web_client - -workflows: - version: 2 - ci: - jobs: - - test - nightly: - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - master - jobs: - - test diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..259926d --- /dev/null +++ b/.env.example @@ -0,0 +1,44 @@ +# Machine-specific paths for script/deploy and the e2e compat harness. +# Copy to .env (gitignored) and edit. Sourced by script/deploy with `set -a`, +# so these are shell assignments — $VAR interpolation works. +# +# Only paths live here. Everything that is logic lives in script/ and e2e/, +# under version control. + +# The Digital Slide Archive devops tree that defines the stack. Not vendorable: +# it is a separate project, and the live mongo + assetstore bind mounts sit +# inside it. +# git clone https://github.com/DigitalSlideArchive/digital_slide_archive +DSA_DEVOPS=/path/to/digital_slide_archive/devops + +# Compose files defining the stack, colon-separated, in layering order. +# script/girder-volview.override.yml must come LAST: it re-points the +# /opt/girder_volview mount at whatever worktree is being deployed, which is the +# only change this repo needs to make to the stack. Keeping it here means the +# upstream DSA checkout stays pristine. +# `docker compose ls` shows what the live project was actually created from — if +# it lists more files, add them or compose sees a different project. +# +# $REPO is this repo's root: script/deploy sets it before sourcing this file. +# Do not use $PWD — .env is sourced without changing directory, so it would +# resolve against wherever you happened to run the command from. +DSA_COMPOSE_FILES=$DSA_DEVOPS/dsa/docker-compose.yml:$DSA_DEVOPS/with-dive-volview/docker-compose.override.yml:$REPO/e2e/seed/docker-compose.minio.yml:$REPO/script/girder-volview.override.yml + +# Must match the running project name. +DSA_COMPOSE_PROJECT=dsa-plus + +# Where VolView checkouts live; a bare argument to script/deploy resolves to +# $VOLVIEW_ROOT/. The compat baseline's VolView sha is recorded in +# e2e/compat-baseline.json. +VOLVIEW_ROOT=/path/to/VolView + +# Where sibling girder_volview worktrees live, for resolving bare branch names +# (girder_volview-). Defaults to this repo's parent directory. The compat +# baseline no longer needs one — it is exported from git history. +#DSA_WORKTREE_ROOT= + +GIRDER_URL=http://localhost:8080 + +# Only needed when the deployed backend ships girder_volview/backend/routes.py, +# which triggers radiology CLI task registration. +CLI_REPO=/path/to/volview-radiology-cli diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..6766178 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,57 @@ +name: CI + +on: + push: + pull_request: + schedule: + # Nightly on the default branch. + - cron: "0 0 * * *" + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + server: + name: Server tests (tox) + runs-on: ubuntu-latest + container: girder/tox-and-node + services: + mongo: + image: mongo:4.4 + options: >- + --tmpfs /data/db + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install tox + run: pip install --no-cache-dir tox + - name: Lint (ruff) + run: tox -e lint + - uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Install the pinned VolView package (carries the backend-contract) + # The conformance suite reads the contract from the `volview` package + # (node_modules/volview/backend-contract), never a vendored copy. + # Runs AFTER ruff so the linter never walks node_modules. + working-directory: girder_volview/web_client + run: npm ci --ignore-scripts --omit=dev + - name: Server tests + run: tox -e test -- -n auto --mongo-uri mongodb://mongo:27017 + + web-client: + name: Web client lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 18 + - name: Install and lint + working-directory: girder_volview/web_client + run: | + npm ci + npm run lint diff --git a/.gitignore b/.gitignore index 45c4b34..a04ec39 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ +# The backend-contract is read from the installed `volview` package +# (node_modules/volview/backend-contract), never vendored. Ignore any local +# materialization someone points GIRDER_VOLVIEW_CONTRACT_DIR at here. +/tests/contract/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -50,3 +55,12 @@ coverage.xml .venv venv/ ENV/ + +# seed-tool downloaded imaging data and MinIO bucket state (see e2e/seed/README.md) +/e2e/seed/data/ +/e2e/seed/.minio-data/ + +# compat harness scratch: baseline checkouts recreated from git history on +# demand (see docs/compat-e2e.md). Never commit old-version sources or the +# session zips they produce — both are reproducible from a sha. +/e2e/.compat/ diff --git a/MANIFEST.in b/MANIFEST.in index 300cc20..0f5cbdf 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,5 +4,6 @@ include setup.py graft girder_volview graft docs +prune girder_volview/web_client/node_modules prune test global-exclude *.py[co] *.cmake __pycache__ node_modules diff --git a/README.md b/README.md index ec6a9f7..c28ec3a 100644 --- a/README.md +++ b/README.md @@ -10,394 +10,45 @@ Open Items in [VolView](https://github.com/Kitware/VolView) with a "Open in VolV - VTK image `.vti` - And many more. Try dragging and dropping the file(s) on the [VolView Demo Site](https://volview.netlify.app/) -## Client Configuration file +## Configuration -Using the client YAML file, anyone can change: +A `.volview_config.yaml` file placed higher in the folder hierarchy configures +the VolView client: view layouts, annotation labels, keyboard shortcuts, +default window/level, segment-group save format, and automatic +layer/segment-group association by file name. -- The default view layout -- Associate files to layer or apply as segmentations via file name -- Default window and level -- Default labels for vector annotation tools +See [Client configuration](./docs/configuration.md), and +[Loading Layers and Segmentations](./docs/loading_layers_and_segmentations.md) +for DICOM-specific association rules. -Add a `.volview_config.yaml` file higher in the folder hierarchy. Example file: +## Sessions: save / restore -```yml -layouts: - Axial: - gridSize: ["axial"] -labels: - defaultLabels: - artifact: - color: "gray" - strokeWidth: 3 - needs-review: - color: "#FFBF00" -``` +Saving in VolView writes a `session.volview.zip` item next to the data; each +save creates a new one, so older saves remain reopenable. Each open gesture has +one meaning: -To merge with `.volview_config.yaml`s higher in the folder hierarchy, include `__inherit__: true` -in the child `.volview_config.yaml` file. Example: +- **Open an item / checked images** → fresh, always. +- **Check a `session.volview.zip` item** → exactly that save (back in history). +- **Open a folder (nothing checked)** → the folder's newest save, else its raw images. +- **Open a grouped DICOM row** → the row's newest save, else its images fresh. +- **Refresh (F5) after saving** → reloads the save you just made. -Child `.volview_config.yaml` +Details, including the launch URL parameters and per-gesture flows, are in +[Sessions](./docs/sessions.md). -```yml -__inherit__: true -shortcuts: - polygon: "Ctrl+p" - rectangle: "b" -``` +## More -Parent `.volview_config.yaml` - -```yml -layouts: - Axial: - gridSize: ["axial"] -``` - -Result - -```yml -shortcuts: - polygon: "Ctrl+p" - rectangle: "b" -layouts: - Axial: - gridSize: ["axial"] -``` - -### Layout Configuration - -Define one or more named layouts using the `layouts` key. -VolView will use the first layout as the default. -Each named layout will appear in the layout selector menu. - -#### Grid with Specific View Types - -Use a 2D array of view type strings to specify both the grid layout and which views appear in each position: - -```yml -layouts: - Four Slice Views: - - [axial, coronal] - - [sagittal, axial] -``` - -Available view types: `axial`, `coronal`, `sagittal`, `volume`, `oblique` - -#### Nested Hierarchical Layout - -For complex layouts, use this nested structure: - -```yml -layouts: - Axial Primary: - direction: row - items: - - axial - - direction: column - items: - - coronal - - sagittal -``` - -Direction values: - -- `row` - items arranged horizontally -- `column` - items stacked vertically - -View object properties: - -- 2D views: `type: 2D`, `orientation: Axial|Coronal|Sagittal`, `name` (optional) -- 3D views: `type: 3D`, `viewDirection` (optional), `viewUp` (optional), `name` (optional) -- Oblique views: `type: Oblique`, `name` (optional) - -#### Multiple Layouts Example - -Define multiple named layouts that users can switch between: - -```yml -layouts: - Three Slice Views: - - [axial, coronal] - - [sagittal, axial] - Axial Focus: - direction: row - items: - - axial - - direction: column - items: - - coronal - - sagittal -``` - -#### Simple Grid (gridSize) - -Alternatively, use `gridSize` to set the layout grid as `[width, height]`: - -```yml -layouts: - Two by Two: - gridSize: [2, 2] -``` - -#### Disabled View Types - -Prevent certain view types from appearing in the view type switcher with this config option. The 3D and Oblique types are disabled by default: - -```yml -disabledViewTypes: - - 3D - - Oblique -``` - -To enable 3D and Oblique views, use an empty list: - -```yml -disabledViewTypes: [] -``` - -Valid values: `2D`, `3D`, `Oblique` - -### Label Configuration - -To assign labels and their properties, add a `.volview_config.yaml` file higher in the folder hierarchy. -Example `.volview_config.yaml` file: - -```yml -# defaultLabels are shared by polygon, ruler and rectangle tool -labels: - defaultLabels: - artifact: - color: "gray" - strokeWidth: 3 - needs-review: - color: "#FFBF00" -``` - -Labels can be configured per tool: - -```yml -labels: - rectangleLabels: - lesion: # label name - color: "#ff0000" - fillColor: "transparent" - innocuous: - color: "white" - fillColor: "#00ff0030" - tumor: - color: "green" - fillColor: "transparent" - - rulerLabels: - big: - color: "#ff0000" - small: - color: "white" -``` - -Label sections could be empty to disable labels for a tool. - -```yml -labels: - rulerLabels: - - rectangleLabels: - lesion: - color: "#ff0000" - fillColor: "transparent" - innocuous: - color: "white" - fillColor: "#00ff0030" -``` - -### Keyboard Shortcuts Configuration - -Configure the keys to activate tools, change selected labels, and more. -Names for shortcut actions are in [constants.ts](https://github.com/Kitware/VolView/blob/main/src/constants.ts#L53) are under the `ACTIONS` variable. - -To configure a key for an action, add its action name and the key(s) under the `shortcuts` section. For key combinations, use `+` like `Ctrl+f`. - -```yml -shortcuts: - polygon: "Ctrl+p" - rectangle: "b" -``` - -In VolView, show a dialog with the configured keyboard shortcuts by pressing the `?` key. - -### Saved Segment Group File Format - -Edited segment groups are saved as separate files within session.volview.zip files.  By default the segment group file format is `nii.gz`. - -```yml -io: - segmentGroupSaveFormat: "nii.gz" # default is nii.gz -``` - -### Automatic Layers and Segment Groups by File Name - -When loading multiple non DICOM image files, VolView can automatically associate related images based on file naming patterns. -The extension must appear anywhere in the filename after splitting by dots, -and the filename must start with the same prefix as the base image (everything before the first dot). - -For example, with a base image `patient.nrrd`: - -- Layers: `patient.layer.1.pet.nii`, `patient.layer.2.ct.mha` -- Segment groups: `patient.seg.1.tumor.nii.gz`, `patient.seg.2.lesion.mha` - -When multiple layers or segment groups match a base image, they are sorted alphabetically by filename and added in that order. - -#### Segment Groups - -Use `segmentGroupExtension` to automatically convert matching non-DICOM images to segment groups. -For example, `myFile.seg.nrrd` becomes a segment group for `myFile.nii`. Defaults to `"seg"`. To disable set to `""`. - -```yml -io: - segmentGroupExtension: "seg" # "seg" is the default -``` - -#### Layering - -Use `layerExtension` to automatically layer matching non-DICOM images on top of the base image. -For example, `myImage.layer.nii` is layered on top of `myImage.nii`. Defaults to `"layer"` .To disable set to `""`. - -```yml -io: - layerExtension: "layer" # "layer" is the default -``` - -### Default Window Level - -Will force the window level for all loaded volumes. - -```yml -windowing: - level: 100 - width: 50 -``` - -## Session Builder - -Generate VolView sessions programmatically with Python. Create sessions with annotations or labelmaps from analysis pipelines. - -See [session_builder/README.md](./session_builder/README.md) for API docs and examples. - -## Customize File Browsing to Group Images and add Columns - -A `.large_image_config.yaml` file can change how images are grouped -and display columns with image metadata. - -[Example YAMLs and docs](./docs/customize_file_browsing.md) - -## Speedup S3 file downloading by disabling proxying - -The VolView plugin proxies request to download files from S3 by default. -This avoids a CORS error when loading a file from an S3 bucket asset store without CORS configuration. -To speed up downloading of files from S3, the Girder admin can: - -1. [Configure CORS](https://girder.readthedocs.io/en/stable/user-guide.html#s3) in the S3 bucket for the Girder server. -2. Change the global [Girder configuration](https://girder.readthedocs.io/en/stable/configuration.html) to add - a `[volview]` section with a `proxy_assetstores = False` option. See below: - -``` -[volview] -# Workaround CORS configuration errors in S3 assetstores. -# If True, the Girder server will proxy file download requests from -# VolView clients to the S3 assetstore. This will use more server bandwidth. -# If False, VolView client requests to download files are redirected to S3. -# Defaults to True. -proxy_assetstores = False -``` - -## API Endpoints - -- GET folder/:id/volview?items=[itemIds]&folders=[folderIds] -> download JSON with URLS to files or the latest `*.volview.zip` file in the folder -- GET item/:id/volview -> download JSON with URLs to all files in item or the latest `*.volview.zip` file -- POST item/:id/volview -> upload file to Item with cookie authentication -- GET file/:id/proxiable/:name -> download a file with option to proxy -- GET folder/:id/volview_config/:name -> download JSON with VolView config properties -- Deprecated: GET item/:id/volview/datasets -> download all files in item except the `*.volview.zip` - -## Example Saving Roundtrip flow - -### Open Item - -1. User clicks Open in VolView for Item - Plugin checks if `*volview.zip` file exists in Item, finds none: - Opens VolView with file download url `item/:id/volview/datasets` -1. VolView opens, fetches from `item/:id/volview/datasets`, receives zip of all files in Item except files ending in `*volview.zip` -1. In VolView, User clicks the Save button - VolView POSTs session.volview.zip to `item/:id/volview` -1. girder_volview plugin saves new session.volview.zip in Item. -1. User clicks Open in VolView for Item - Plugin finds a `*volview.zip` in the Item. Opens VolView with file download URL pointing to `item/:id/volview` -1. VolView opens, fetches from `item/:id/volview`, receives most recently created `*volview.zip` file in Item. - -VolView creates a new session.volview.zip file in the Girder Item every time the Save button is clicked. - -### Open Checked - -1. User checks a set of items or folders. Clicks "Open Checked in VolView". -1. Browser client updates the `lastOpened` metadata on a checked item/folder metadata with the current time. -1. Browser opens VolView with file download url pointing to `GET folder/:id/volview?items=[...ids]&folders=[...ids]`. That endpoint returns a JSON file with URLs to Girder files. -1. VolView save URL is pointing to `PUT folder/:id/volview?metadata={items: [...ids], folders: [...ids]}`. `metadata` parameter matches the checked set in the Girder file browser. User clicks save. `session.volview.zip` item is created in the folder with a `linkedResources` metadata key holding the folder and item IDs. If user checked a session.volview.zip item, then `items` points to an existing session.volview.zip. The new session.volview.zip takes the `linkedResources` of the older session.volview.zip. -1. If user clicks refresh in VolView, the `GET folder/:id/volview?items=[...ids]&folders=[...ids]` end point is hit again. If a session.volview.zip is in the `items` parameter, the plugin reads the volview.zip's `linkedResources` and searches for a newer session.volview.zips with matching `linkedResources` and returns that if found. -1. If user checks a new set of folders or items that does not include a session.volview.zip item, the `GET folder/:id/volview` endpoint does not pick a session.volview.zip with matching `linkedResources` as `lastOpened` metadata on one of the checked items/folders is newer than the matching session.volview.zip. This allows opening of images with a clean slate. - -### Open Filter-Linked Session (Grouped DICOM Row) - -Filter-linked sessions use `linkedResources.filter` (a metadata-key/value dict like `{"meta.dicom.StudyInstanceUID": "..."}`) in place of explicit item/folder IDs. The grouped DICOM row opener produces these. - -1. User clicks Open on a grouped row. Browser opens VolView with a manifest URL of `GET folder/:id/volview?filters={...}`. The endpoint returns the newest session.volview.zip whose `linkedResources.filter` is *equal* to the row's filter (strict set-equality on the filter list); if none exists, it returns the raw DICOM files matching the filter. -1. User clicks Save. A new session.volview.zip is created in the folder with the row's filter recorded under `linkedResources.filter`. -1. User clicks refresh in VolView. The same `?filters={...}` URL is hit again and now resolves to the just-saved session (newest matching by `getTouchedTime`, which honors `meta.lastOpened`). -1. User checks an older filter-linked session item in the file browser and clicks "Open Checked in VolView". Client bumps `lastOpened` on the checked item, then opens VolView with `?items=`. The endpoint reads the checked session's `linkedResources.filter` and returns the newest matching session — which is the just-touched older one. Subsequent saves create newer matching sessions; refresh picks them up via the same touched-time rule. This is what makes "go back in history" and "refresh after save" use the same code path. +- [Session Builder](./session_builder/README.md) — generate VolView sessions + programmatically with Python, e.g. from analysis pipelines. +- [VolView Radiology CLI](https://github.com/PaulHax/volview-radiology-cli) — + reference task image used to drive Girder-VolView processing in development + and end-to-end tests. +- [Customize file browsing](./docs/customize_file_browsing.md) — group images + and add metadata columns via `.large_image_config.yaml`. +- [Server administration](./docs/admin.md) — S3 download proxying. ## Development -Get this running https://github.com/DigitalSlideArchive/digital_slide_archive/tree/master/devops/with-dive-volview - -In the `docker-compose.override.yml` file, add some `volumes` pointing to this girder plugin and optionally -a VolView repo checkout. Example: - -```yaml -services: - girder: - volumes: - - ../with-dive-volview/provision.divevolview.yaml:/opt/digital_slide_archive/devops/dsa/provision.yaml - - ../../../girder_volview:/opt/girder_volview - - ../../../../VolView:/opt/volview-package -``` - -Comment out the pip install of this plugin here: https://github.com/DigitalSlideArchive/digital_slide_archive/blob/master/devops/with-dive-volview/provision.divevolview.yaml#L3 - -To install volume mapped girder-volview plugin and incorporate changes as files are edited, add this to the `shell` section of the provision.yaml: - -```yaml -shell: - - cd /opt/girder_volview/ && pip install -e . - - (sleep 30 && girder build --dev --watch-plugin volview)& -``` - -### Develop VolView client - -To develop with a local VolView build, change the directory the Webpack copy plugin pulls from in `girder_volview/web_client/webpack.helper.js`: - -```js -new CopyWebpackPlugin([ - { - from: "/opt/volview-package/dist", // Point to your mount of VolView - to: config.output.path, - toType: "dir", - }, -]); -``` - -Then build VolView from source with these env vars: - -```sh -VITE_ENABLE_REMOTE_SAVE=true npm run build -``` - -### Updating the VolView Client Version - -1. Update the [volview](https://www.npmjs.com/package/volview?activeTab=versions) version in `./girder_volview/web_client/package.json` +Dev-stack setup, API endpoints, developing the VolView client against this +plugin, and the backend contract/conformance tests are documented in +[Development](./docs/development.md). diff --git a/docs/admin.md b/docs/admin.md new file mode 100644 index 0000000..0d9cb6f --- /dev/null +++ b/docs/admin.md @@ -0,0 +1,47 @@ +# Server administration + +## Job processing + +See [Job processing design](./job-processing.md) for the processing +architecture, job execution flow, Slicer CLI registration guidance, and the +category scoping that keeps VolView tasks separate from HistomicsUI and +DIVE-DSA tasks in a shared `slicer_cli_web` deployment. + +### Radiology CLI task image + +The development stack uses the +[VolView Radiology CLI](https://github.com/PaulHax/volview-radiology-cli) as +reference infrastructure to drive and test the processing backend. Clone it +locally and set `CLI_REPO` in this repository's `.env` to that checkout: + +```sh +git clone https://github.com/PaulHax/volview-radiology-cli +# In girder_volview/.env: +CLI_REPO=/path/to/volview-radiology-cli +``` + +When processing routes are present, `script/deploy` calls +`script/ensure-radiology-cli`. That script builds the local +`volview-radiology-cli:latest` image if it is missing, registers it with +`slicer_cli_web`, and verifies the declared tasks are available. It does not +pull this image from a registry. + +## Speedup S3 file downloading by disabling proxying + +The VolView plugin proxies request to download files from S3 by default. +This avoids a CORS error when loading a file from an S3 bucket asset store without CORS configuration. +To speed up downloading of files from S3, the Girder admin can: + +1. [Configure CORS](https://girder.readthedocs.io/en/stable/user-guide.html#s3) in the S3 bucket for the Girder server. +2. Change the global [Girder configuration](https://girder.readthedocs.io/en/stable/configuration.html) to add + a `[volview]` section with a `proxy_assetstores = False` option. See below: + +``` +[volview] +# Workaround CORS configuration errors in S3 assetstores. +# If True, the Girder server will proxy file download requests from +# VolView clients to the S3 assetstore. This will use more server bandwidth. +# If False, VolView client requests to download files are redirected to S3. +# Defaults to True. +proxy_assetstores = False +``` diff --git a/docs/compat-e2e.md b/docs/compat-e2e.md new file mode 100644 index 0000000..fdb8a94 --- /dev/null +++ b/docs/compat-e2e.md @@ -0,0 +1,156 @@ +# Browser lifecycle and backwards-compatibility e2e + +This is the project's single browser-test infrastructure. It proves that +`session.volview.zip` files saved by an **older** girder_volview + VolView client +still restore and re-save correctly, then exercises fresh current-version +save/load/restore and job behavior. + +``` +e2e/scripts/compat.sh + ├─ materialize-baseline.sh # git archive -> e2e/.compat/ + ├─ script/deploy main # baseline backend + client + ├─ playwright --project capture # save sessions on the baseline + ├─ script/deploy just-jobs # branch backend + client + ├─ playwright --project verify # old sessions must restore + └─ playwright --project current # fresh lifecycle + jobs +``` + +Neither the old sources nor the session zips are committed — both are +reproducible from a sha. The repo stores a pointer (`e2e/compat-baseline.json`) +and the harness recreates the rest into the gitignored `e2e/.compat/`. + +Girder's mongo volume survives the redeploy (script/deploy only recreates the +girder container's code), so the folders and sessions captured in step 2 are +still there for step 4. `e2e/.compat-state.json` (gitignored) bridges the two +playwright invocations: session item ids, launch descriptors, and the expected +content per gesture. + +## What the harness proves + +| Gesture | Launch (real girder UI, on the baseline) | Content saved | Verified on the branch | +| ------------------------- | ----------------------------------------------------------- | ----------------------------- | -------------------------------------------------------------------------------------------------------------- | +| `single-item` | item page → Open in VolView | ruler | item manifest serves the session; measurements exact; re-save | +| `checked-nrrd` | check 2 NRRD rows → Open Checked | ruler + painted segment group | bare-folder open resumes it; checking the session row opens exactly it; groups + measurements survive; re-save | +| `filtered-dicom` | filter box narrows to one patient → check series row → Open | ruler | replaying the same filter gesture resumes the matching `session..volview.zip` | +| `study-layered` | check CT + PET series rows of one study → Open | PET layered over CT + ruler | replay resumes; the layer survives restore and re-save | +| `devkit-study` (optional) | patient → study drill-down in the devkit collection | PET layer + ruler | replay resumes; skipped unless `seed.py seed` has run | + +Content checks are semantic, not pixel-based: ruler measurement text must match +exactly, segment-group names must survive, and the re-saved manifest must keep +rulers/segment groups/layers (schema migrations are fine; content loss is not). +Screenshots are attached to the report as evidence, never asserted on. + +The current project also covers what an old-session restore can't prove: + +- fresh single-item, checked-image, and grouped-filter launches; +- F5 before save staying fresh, and F5 after first and second saves resuming; +- checked raw images deliberately restarting instead of resuming; +- bare-folder newest-session selection and exact older-session selection; +- the launch button's `urls`, `save`, `config`, and `names` contract; +- saved-session rows, completed-job loading, and live job auto-apply. + +Each scenario owns a folder, so state can't leak between tests. Tests run on +one worker for predictable load, but aren't Playwright `serial` — one failure +doesn't mark the rest as unrun. + +The DICOM fixtures use real IDC data (ACRIN NSCLC FDG-PET/CT, CC-BY): the +devkit's pinned `patient-01/study-01/{CT,PET}` + `patient-02/study-01/CT` +series at 12 slices each. Each grouped scenario gets its own uploaded copy, +with `meta.dicom.*` populated by girder_volview, plus +`e2e/fixtures/dicom.large_image_config.yaml` so the folder groups into +filterable series rows. + +## Running it + +One-time setup: + +```bash +cd e2e && npm ci && npm run install-browser +uv run seed/seed.py fetch --small # small DICOM cache (also run by compat.sh) +``` + +Machine-specific paths live in a gitignored `.env` at the repo root: + +```bash +cp .env.example .env && $EDITOR .env # DSA_DEVOPS, VOLVIEW_ROOT, ... +``` + +The stack must already be running (see `docs/development.md`) — `script/deploy` +only swaps the code it serves. `script/girder-volview.override.yml` re-points +the `/opt/girder_volview` mount at the worktree (or compat baseline) being +deployed; that's the only change this repo makes to the upstream stack. + +What the harness needs beyond `.env`: + +- **The baseline backend** — no checkout required. It's exported from this + repo's own history at the sha pinned in `e2e/compat-baseline.json`. Override + for a one-off with `COMPAT_BASELINE_REF=origin/main`, or point at a real git + checkout with `COMPAT_OLD_CHECKOUT` + `COMPAT_OLD_SHA` (HEAD must equal that + sha). +- **VolView worktrees** `main` and `just-jobs` under `VOLVIEW_ROOT` (override + `COMPAT_BASELINE_VOLVIEW` / `COMPAT_BRANCH_VOLVIEW`). Baseline sha and + published npm version are pinned in `e2e/compat-baseline.json`; the + branch-side client is unpublished and builds from source, with its sha read + from the worktree at run start. Set `COMPAT_BRANCH_VOLVIEW_SHA` to assert the + checkout is at one particular commit. + +To move the baseline forward, resolve the new sha and edit +`e2e/compat-baseline.json` — it's pinned rather than floating so a red compat +run is bisectable. + +Full coverage-first run (two deploys): + +```bash +cd e2e && npm test +``` + +`npm run compat` is an alias for the same run. + +Iterating: + +```bash +npm run compat:verify-fast # verify only, no redeploy, keep state for re-runs +npm run compat:capture # capture half only (deploys main first) +bash scripts/compat.sh --phase capture --skip-deploy # re-capture, baseline already deployed +bash scripts/compat.sh --phase current --skip-deploy # current scenarios using retained state +bash scripts/compat.sh --link # fast client deploys (docker cp, no npm pack) +COMPAT_BRANCH_VOLVIEW=/abs/path/to/VolView/just-jobs npm test # another current checkout +npm run compat:clean # remove materialized baselines (handles root-owned residue) +npm run report # html report of the last phase +``` + +The capture phase refuses to start if `.compat-state.json` already exists, +meaning a previous capture was never verified or torn down. Either finish it +(`npm run compat:verify`) or delete the state file and the +`girder-volview-compat-` folder it names. + +Optional full-devkit tier: seed the "VolView Devkit" collection first +(`e2e/seed/README.md`); the `devkit-study` gesture then runs automatically and +cleans up the session items it mints. + +## Guards + +- `verifyDeployedHeads` (e2e/helpers/stack.ts) requires the deployed backend + and client shas to equal the intended checkouts. Capture runs against the + baseline pair, so `compat.sh` passes `E2E_EXPECT_GIRDER_SHA` and + `E2E_EXPECT_VOLVIEW_SHA`; verify checks this backend's HEAD and the current + client HEAD. Outside compat it checks this worktree and the receipt's live + VolView checkout. A missing or unreadable expected sha is a hard failure. +- The guard also hashes the served `index.html` and, when the receipt + worktrees are locally accessible, compares the built client and Python + source tree to the receipt — moving a checkout or changing code after deploy + forces a fresh deploy instead of a false green. +- `script/deploy` writes the receipt only after confirming the served SPA and + mounted backend match what it deployed, so a receipt never certifies a + partial deploy. +- The baseline export has no `.git`, so its sha is asserted via `--girder-sha` + and backstopped by comparing the mounted backend tree hash to the tree + `materialize-baseline.sh` produced. +- One `playwright.config.ts` defines the capture, verify, and current + projects, and refuses to start without the phase `compat.sh` selects — + preventing a partial direct invocation from silently testing the wrong + deploy. +- CI does not run this harness; it's a local tool by design. A future job + needs the full DSA Compose stack plus a separate VolView checkout via + `COMPAT_BRANCH_VOLVIEW` (and `COMPAT_BRANCH_VOLVIEW_SHA` for a reproducible + cross-repository assertion). diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..5eb0a97 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,277 @@ +# Client Configuration file + +This covers `.volview_config.yaml`, which configures the VolView client itself. +To group images and add metadata columns in Girder's file browser, see +[Customize file browsing](./customize_file_browsing.md), which uses a +separate `.large_image_config.yaml` file. + +Using the client YAML file, anyone can change: + +- The default view layout +- Associate files to layer or apply as segmentations via file name +- Default window and level +- Default labels for vector annotation tools + +Add a `.volview_config.yaml` file higher in the folder hierarchy. Example file: + +```yml +layouts: + Axial: + gridSize: ["axial"] +labels: + defaultLabels: + artifact: + color: "gray" + strokeWidth: 3 + needs-review: + color: "#FFBF00" +``` + +To merge with `.volview_config.yaml`s higher in the folder hierarchy, include `__inherit__: true` +in the child `.volview_config.yaml` file. Example: + +Child `.volview_config.yaml` + +```yml +__inherit__: true +shortcuts: + polygon: "Ctrl+p" + rectangle: "b" +``` + +Parent `.volview_config.yaml` + +```yml +layouts: + Axial: + gridSize: ["axial"] +``` + +Result + +```yml +shortcuts: + polygon: "Ctrl+p" + rectangle: "b" +layouts: + Axial: + gridSize: ["axial"] +``` + +## Layout Configuration + +Define one or more named layouts using the `layouts` key. +VolView will use the first layout as the default. +Each named layout will appear in the layout selector menu. + +### Grid with Specific View Types + +Use a 2D array of view type strings to specify both the grid layout and which views appear in each position: + +```yml +layouts: + Four Slice Views: + - [axial, coronal] + - [sagittal, axial] +``` + +Available view types: `axial`, `coronal`, `sagittal`, `volume`, `oblique` + +### Nested Hierarchical Layout + +For complex layouts, use this nested structure: + +```yml +layouts: + Axial Primary: + direction: row + items: + - axial + - direction: column + items: + - coronal + - sagittal +``` + +Direction values: + +- `row` - items arranged horizontally +- `column` - items stacked vertically + +View object properties: + +- 2D views: `type: 2D`, `orientation: Axial|Coronal|Sagittal`, `name` (optional) +- 3D views: `type: 3D`, `viewDirection` (optional), `viewUp` (optional), `name` (optional) +- Oblique views: `type: Oblique`, `name` (optional) + +### Multiple Layouts Example + +Define multiple named layouts that users can switch between: + +```yml +layouts: + Three Slice Views: + - [axial, coronal] + - [sagittal, axial] + Axial Focus: + direction: row + items: + - axial + - direction: column + items: + - coronal + - sagittal +``` + +### Simple Grid (gridSize) + +Alternatively, use `gridSize` to set the layout grid as `[width, height]`: + +```yml +layouts: + Two by Two: + gridSize: [2, 2] +``` + +### Disabled View Types + +Prevent certain view types from appearing in the view type switcher with this config option. The 3D and Oblique types are disabled by default: + +```yml +disabledViewTypes: + - 3D + - Oblique +``` + +To enable 3D and Oblique views, use an empty list: + +```yml +disabledViewTypes: [] +``` + +Valid values: `2D`, `3D`, `Oblique` + +## Label Configuration + +To assign labels and their properties, add a `.volview_config.yaml` file higher in the folder hierarchy. +Example `.volview_config.yaml` file: + +```yml +# defaultLabels are shared by polygon, ruler and rectangle tool +labels: + defaultLabels: + artifact: + color: "gray" + strokeWidth: 3 + needs-review: + color: "#FFBF00" +``` + +Labels can be configured per tool: + +```yml +labels: + rectangleLabels: + lesion: # label name + color: "#ff0000" + fillColor: "transparent" + innocuous: + color: "white" + fillColor: "#00ff0030" + tumor: + color: "green" + fillColor: "transparent" + + rulerLabels: + big: + color: "#ff0000" + small: + color: "white" +``` + +Label sections could be empty to disable labels for a tool. + +```yml +labels: + rulerLabels: + + rectangleLabels: + lesion: + color: "#ff0000" + fillColor: "transparent" + innocuous: + color: "white" + fillColor: "#00ff0030" +``` + +## Keyboard Shortcuts Configuration + +Configure the keys to activate tools, change selected labels, and more. +Names for shortcut actions are in [constants.ts](https://github.com/Kitware/VolView/blob/main/src/constants.ts#L53) are under the `ACTIONS` variable. + +To configure a key for an action, add its action name and the key(s) under the `shortcuts` section. For key combinations, use `+` like `Ctrl+f`. + +```yml +shortcuts: + polygon: "Ctrl+p" + rectangle: "b" +``` + +In VolView, show a dialog with the configured keyboard shortcuts by pressing the `?` key. + +## Saved Segment Group File Format + +Edited segment groups are saved as separate files within session.volview.zip files.  By default the segment group file format is `nii.gz`. + +```yml +io: + segmentGroupSaveFormat: "nii.gz" # default is nii.gz +``` + +## Automatic Layers and Segment Groups by File Name + +When loading multiple image files, VolView can automatically associate related images based on file naming patterns. +For non-DICOM base images, the matching rule is based on the base filename prefix. +The extension must appear anywhere in the filename after splitting by dots, +and the filename must start with the same prefix as the base image (everything before the first dot). + +For example, with a base image `patient.nrrd`: + +- Layers: `patient.layer.1.pet.nii`, `patient.layer.2.ct.mha` +- Segment groups: `patient.seg.1.tumor.nii.gz`, `patient.seg.2.lesion.mha` + +When multiple layers or segment groups match a base image, they are sorted alphabetically by filename and added in that order. + +### Segment Groups + +Use `segmentGroupExtension` to automatically convert matching non-DICOM images to segment groups. +For example, `myFile.seg.nrrd` becomes a segment group for `myFile.nii`. Defaults to `"seg"`. To disable set to `""`. + +```yml +io: + segmentGroupExtension: "seg" # "seg" is the default +``` + +### Layering + +Use `layerExtension` to automatically layer matching non-DICOM images on top of the base image. +For example, `myImage.layer.nii` is layered on top of `myImage.nii`. Defaults to `"layer"` .To disable set to `""`. + +```yml +io: + layerExtension: "layer" # "layer" is the default +``` + +For DICOM-specific association rules, explicit `segmentGroups` / +`parentToLayers` session manifest examples, and notes on using DICOM tags versus +file names, see [Loading Layers and Segmentations](./loading_layers_and_segmentations.md). + +## Default Window Level + +Will force the window level for all loaded volumes. + +```yml +windowing: + level: 100 + width: 50 +``` diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..9c59ca4 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,129 @@ +# Development + +## Dev stack + +Get a stack running with +https://github.com/DigitalSlideArchive/digital_slide_archive/tree/master/devops/with-dive-volview + +In the `docker-compose.override.yml` file, add a volume mounting your checkout +of this plugin (replace `/path/to/girder_volview` with wherever you cloned it — +the paths make no assumption about checkouts sitting next to each other): + +```yaml +services: + girder: + volumes: + - ../with-dive-volview/provision.divevolview.yaml:/opt/digital_slide_archive/devops/dsa/provision.yaml + - /path/to/girder_volview:/opt/girder_volview +``` + +Comment out the pip install of this plugin here: https://github.com/DigitalSlideArchive/digital_slide_archive/blob/master/devops/with-dive-volview/provision.divevolview.yaml#L3 + +To install the volume-mapped plugin and incorporate changes as files are +edited, add this to the `shell` section of the provision.yaml: + +```yaml +shell: + - cd /opt/girder_volview/ && pip install -e . + - (sleep 30 && girder build --dev --watch-plugin volview)& +``` + +## API endpoints + +- GET folder/:id/volview?items=[itemIds]&folders=[folderIds] -> download JSON with URLS to files or the latest `*.volview.zip` file in the folder +- GET item/:id/volview -> download JSON with URLs to all files in item or the latest `*.volview.zip` file +- POST item/:id/volview -> upload file to Item with cookie authentication +- GET file/:id/proxiable/:name -> download a file with option to proxy +- GET folder/:id/volview_config/:name -> download JSON with VolView config properties + +The launch-manifest routes' resume/fresh semantics are documented in +[sessions.md](./sessions.md). + +## Develop the VolView client + +The VolView client is consumed as the `volview` npm package: `girder build` +installs the version pinned in `girder_volview/web_client/package.json` and +serves its `dist/`; the backend conformance tests read the same package's +`backend-contract/`. To develop against an unreleased VolView build, make +`girder_volview/web_client/node_modules/volview` BE your local build instead +of the pinned release — either `npm link` your VolView checkout, or +`npm pack` it and install the tarball (the closer match to what a release +does, since it goes through the package's `files` allowlist): + +```sh +cd /path/to/VolView && npm run build && npm pack +npm --prefix girder_volview/web_client install /path/to/VolView/volview-*.tgz +``` + +Then rebuild/restart girder so the served dist is refreshed. Mounting or +copying only a `dist/` over `node_modules/volview/dist` also works for +UI-only iteration, but leaves the package's `backend-contract` at the pinned +version — fine for the browser, wrong for the conformance tests. + +Processing (the Jobs tab) and remote session save ship in every build +and no longer need build-time env flags — `VITE_ENABLE_PROCESSING`, +`VITE_ENABLE_REMOTE_SAVE`, and `VITE_PROCESSING_ALLOWED_ORIGINS` were removed. +What the deployed client is allowed to contact is decided at runtime by a +same-origin egress gate; a same-origin deployment (such as DSA) needs no +configuration, and cross-origin targets are never allowed. + +See [Job processing design](./job-processing.md) for the execution flow and how +to add a task using the +[VolView Radiology CLI](https://github.com/PaulHax/volview-radiology-cli) as a +reference implementation. Image build and registration instructions are in +[Server administration](./admin.md#radiology-cli-task-image). + +## Updating the VolView client version + +1. Update the [volview](https://www.npmjs.com/package/volview?activeTab=versions) version in `./girder_volview/web_client/package.json` + +## Backend contract and tests + +The backend's conformance tests validate the server against VolView's +`backend-contract` — the ONE normative copy of the wire fixtures + generated +JSON Schemas. This repo keeps **no vendored copy**; the tests read the contract +from wherever the `volview` dependency is installed +(`girder_volview/web_client/node_modules/volview/backend-contract`, shipped in +the package's `files`). + +- **Against the pinned release** — fetch the pinned `volview`, then run the + suite: + + ```sh + npm --prefix girder_volview/web_client install + tox -e test # or: pytest + ``` + +- **Against an unreleased VolView branch** (developing the two together) — link + a local VolView checkout so the tests read that branch's contract: + + ```sh + cd && npm link + npm --prefix girder_volview/web_client link volview + ``` + + Or point the tests straight at a checkout, no link required: + + ```sh + GIRDER_VOLVIEW_CONTRACT_DIR=/backend-contract pytest + ``` + +CI installs the **pinned published** `volview`, so when the backend is developed +ahead of the latest published contract the conformance tests are expected to be +red; they go green once VolView publishes (a merge-to-main dev release) and the +`volview` pin here is bumped to it. + +## Browser e2e harness + +`e2e/` has one Playwright harness. `npm test` exports and deploys the pinned +baseline, captures sessions through its real UI, redeploys this worktree, +verifies backwards compatibility, and then runs fresh current-version +save/load/restore and job scenarios. `npm run compat` is an alias for the same +coverage-first run. No second backend checkout or committed session fixture is +required. See [compat-e2e.md](compat-e2e.md). + +Both deploy through `script/deploy`, which reads machine-specific paths from a +gitignored repo-root `.env` (copy `.env.example`). + +Sample-data tooling (including the harness's cached DICOM tier) lives in +[`e2e/seed/`](../e2e/seed/README.md). diff --git a/docs/job-processing.md b/docs/job-processing.md new file mode 100644 index 0000000..ef720ad --- /dev/null +++ b/docs/job-processing.md @@ -0,0 +1,90 @@ +# Job processing design + +This branch adds server-backed processing to VolView's Jobs tab. The +Girder VolView plugin presents a small, VolView-specific API while continuing to +use `slicer_cli_web`, Girder Jobs, Girder Worker, and containerized Slicer CLIs +for task registration and execution. + +At a high level, the pieces are: + +- **VolView** discovers tasks, renders their parameters, submits work, polls job + status, and applies completed results to the active scene. +- **Girder VolView** translates between VolView's processing contract and the + Slicer CLI model. It validates submissions and owns the job-to-output + relationship. +- **`slicer_cli_web`** registers the CLIs supplied by container images and + creates the Docker-backed Girder jobs. +- **Girder Worker** runs the selected container and transfers its inputs and + outputs through Girder. + +## Execution flow + +When a user submits a task, the following happens: + +1. VolView gets the available task list for the folder that provided its launch + context. The backend reads the accessible `slicer_cli_web` catalog and + returns only CLIs in the configured VolView processing categories. +2. VolView requests the selected task's specification. Girder VolView parses + the Slicer Execution Model XML and translates it into the task and parameter + shapes understood by VolView. +3. VolView submits the selected task ID and parameter values. Inputs already in + Girder are represented by file handles; client-generated data is first + staged into the launch folder's server-owned `volview-jobs` container through + the processing API, keeping temporary working items out of the source-data + folder. +4. Girder VolView authenticates the user, checks folder access, confirms that + the task is still in VolView's allowed scope, and validates every submitted + value against the CLI declaration. Reserved credentials, undeclared + parameters, and caller-selected output locations are rejected. +5. The backend creates a private output folder for this submission. It resolves + input handles using the submitting user's permissions, copies transient + staged inputs into the job-owned folder, generates safe output names, and + forces every declared output into that folder. +6. The backend asks `slicer_cli_web` to create the container job. The job record + is created with its launch context, submitted parameters, declared outputs, + transient inputs, and owned output-folder ID before the task is published to + Girder Worker. +7. Girder Worker starts the registered container. The CLI fetches its inputs + from Girder using a short-lived, scoped token, performs the operation, and + uploads its declared outputs back to the private output folder. +8. As each upload is finalized, Girder VolView associates the file with the job + using the job-owned folder and the worker's declared output reference. It + does not correlate results by filename, so concurrent jobs producing the + same filename cannot cross-associate their results. +9. VolView polls the job-addressed status and results endpoints. The backend + projects Girder's job states into the VolView contract and returns completed + files as result records with declarative application intents, such as adding + a base image or segment group. +10. VolView applies ready results to the scene. The completed job remains in the + user's folder-scoped history and can be reopened later. Transient inputs are + cleaned up when execution settles; deleting a terminal job also deletes its + owned output folder and results. + +## Adding a Slicer CLI + +Use the +[VolView Radiology CLI repository](https://github.com/PaulHax/volview-radiology-cli) +as the reference implementation. A new operation needs a Slicer Execution Model +XML description, an executable that follows that description, and an entry in +the image's `cli_list.json`. Build and register the image as described in +[development.md](./development.md#radiology-cli-task-image); Girder VolView +derives the task form and output handling from the XML rather than requiring +task-specific backend code. + +Choose a unique Docker image name and unique CLI executable/task names. Do not +reuse an image or task identity already registered by HistomicsUI or DIVE-DSA: +all three applications can share the same `slicer_cli_web` task folder, and +registration must not replace another application's entries. + +The CLI XML must also declare a category intended for VolView. By default, +Girder VolView admits `Radiology`, `Segmentation`, and `Filtering`, matched +case-insensitively. The allowed set can be changed with +`VOLVIEW_PROCESSING_ALLOWED_CATEGORIES`. HistomicsUI's `HistomicsTK` pathology +CLIs, DIVE-DSA CLIs in other categories, uncategorized CLIs, and malformed +descriptions are excluded from VolView's task list. The same category check is +performed again for task-spec requests and job submission, so an out-of-scope +task cannot be invoked through VolView by guessing its ID. + +These rules provide two separate protections: unique image/task identities +avoid collisions in the shared registration catalog, while category scoping +keeps each application's CLIs out of the other application's user interface. diff --git a/docs/loading_layers_and_segmentations.md b/docs/loading_layers_and_segmentations.md new file mode 100644 index 0000000..113655b --- /dev/null +++ b/docs/loading_layers_and_segmentations.md @@ -0,0 +1,57 @@ +# Loading Layers and Segmentations + +VolView can load a base image with two overlay types: + +- Layer: scalar image data such as PET, perfusion, probability, or heat maps. +- Segmentation: integer label maps shown in the Segment Groups panel. + +The overlay must overlap the base image in physical space. VolView resamples it +into the base image space. + +## Automatic Filename Matching + +By default, use `.seg.` for segmentations and `.layer.` for layers: + +```text +patient01.ct.nii.gz # base +patient01.seg.tumor.nii.gz # segmentation +patient01.layer.pet.nii.gz # layer +``` + +For NIfTI/NRRD/MHA files: + +- The base prefix is the text before the first dot. +- Overlays must start with the same prefix. +- The configured extension must appear as a dot-separated token. +- Multiple matches load alphabetically. + +To change the tokens, put `.volview_config.yaml` at or above the Girder folder: + +```yaml +io: + segmentGroupExtension: "seg" + layerExtension: "layer" +``` + +Set either value to `""` to disable that automatic conversion. + +## DICOM Matching + +VolView groups DICOM instances into volumes, then chooses a preferred base. + +- If the selected base is CT, the first PT volume in the same + `StudyInstanceUID` is added as a layer. +- DICOM SEG volumes in the same `StudyInstanceUID` become segmentations. +- Non-DICOM overlays match a DICOM base by `SeriesNumber`. + +Example: + +```text +CT SeriesNumber = 3 +3.seg.tumor.nii.gz +3.layer.pet.nii.gz +``` + +## Explicit Manifests + +Use a VolView session manifest JSON for lower level control. [Session builder README](../session_builder/README.md) diff --git a/docs/sessions.md b/docs/sessions.md new file mode 100644 index 0000000..db13958 --- /dev/null +++ b/docs/sessions.md @@ -0,0 +1,44 @@ +# Save / restore round-trip + +Every launch URL carries query params the client acts on: + +- `urls=` — where the client fetches the scene to load. Each gesture has one + meaning: a **raw pick** (an item or checked images) always loads fresh; a + **checked session item** opens exactly that saved session (back-in-history); + a **filter gesture** (grouped DICOM row) resumes its newest matching session, + else the filtered images fresh; a **bare folder open** resumes the folder's + newest `session.volview.zip`, or the folder's raw images if none has been + saved yet. +- `save=` — the ordinary session-zip save route: item-scoped + (`POST item/:id/volview`) for a single item, or folder-scoped + (`POST folder/:id/volview?metadata=…`) for a checked or filter set, where + `metadata` records that set under the saved session's `linkedResources`. +- `config=` — the folder's VolView config (`GET folder/:id/volview_config/:name`). + +On Save, the plugin writes a `session.volview.zip` and returns a **`resumeUrl`** +(`item/:id/volview`, pointing at the session item). The client repoints ONLY its +`urls=` at that `resumeUrl` — `save=` stays as launched — so: + +- a browser refresh (F5) reloads the just-made save directly from its item, and +- repeated folder-scoped saves each mint a **new** `session.volview.zip` item in + the folder; F5 and a bare folder re-open track the newest. + +## Open Item + +1. User clicks Open in VolView for an item. If the item has no `session.volview.zip`, VolView opens on the item's raw files; if it has one, VolView opens on the newest `session.volview.zip`. +1. User clicks Save. VolView POSTs `session.volview.zip` to `item/:id/volview`; the plugin stores it in the item and returns the `resumeUrl`. +1. Refresh, or re-open, resumes that saved session from the same item. + +## Open Checked + +1. User checks a set of items/folders and clicks "Open Checked in VolView". VolView opens fresh on exactly the checked set (`GET folder/:id/volview?items=[…]&folders=[…]`) — checking raw images ALWAYS opens fresh, even when a newer matching save exists. Checking a `session.volview.zip` item instead opens exactly that saved session (back-in-history). +1. User clicks Save. A `session.volview.zip` is created in the folder with the checked set recorded under `linkedResources`; the plugin returns its `resumeUrl`, which the client repoints `urls=` at. +1. Refresh reloads that saved session; each subsequent save mints a new session item in the folder. To get back to a save later, open the folder bare (newest save) or check the session item itself (that save). + +## Open Filter-Linked Session (Grouped DICOM Row) + +Filter-linked sessions record a `linkedResources.filter` (a metadata key/value dict like `{"meta.dicom.StudyInstanceUID": "..."}`) in place of explicit item/folder IDs. The grouped DICOM row opener produces these. + +1. User clicks Open on a grouped row. If a session with a matching filter exists, VolView resumes the newest one; otherwise it opens fresh on the raw DICOM files matching the filter (`GET folder/:id/volview?filters={…}`). +1. User clicks Save. A `session.volview.zip` is created in the folder with the row's filter recorded under `linkedResources.filter`; the client repoints `urls=` at its `resumeUrl`. +1. Refresh reloads the saved session; each subsequent save mints a new session item in the folder. diff --git a/e2e/.gitignore b/e2e/.gitignore new file mode 100644 index 0000000..0f270fe --- /dev/null +++ b/e2e/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +test-results/ +playwright-report/ +playwright/.cache/ +.compat-state.json diff --git a/e2e/compat-baseline.json b/e2e/compat-baseline.json new file mode 100644 index 0000000..81a6f33 --- /dev/null +++ b/e2e/compat-baseline.json @@ -0,0 +1,32 @@ +{ + "_comment": [ + "The version pair the compat suite reads old sessions from. Nothing here is", + "a copy of old code — it is a pointer. e2e/scripts/materialize-baseline.sh", + "recreates the baseline girder_volview tree from git history on demand into", + "the gitignored e2e/.compat/, and the sessions it produces are regenerated", + "every run rather than committed.", + "", + "Pinned rather than floating so a red compat run is bisectable: 'did main", + "move, or did I break it?' has an answer. Run with COMPAT_BASELINE_REF=", + "to override for a one-off freshness check against a moving branch." + ], + + "girder": { + "ref": "origin/main", + "sha": "061b20af647a6b36787ee3b3d63edac928e948b1", + "_comment": "Resolved from `ref` at pin time. materialize-baseline.sh exports THIS sha." + }, + + "volview": { + "repo": "https://github.com/PaulHax/VolView.git", + "sha": "9557e2658c4a6b428c430adb7b100d143096274b", + "npmVersion": "4.4.2-dev.9557e2658c4a6b428c430adb7b100d143096274b.0", + "_comment": [ + "The client paired with the baseline backend. This is also the version", + "girder_volview/web_client/package.json pins today, so it is published and", + "an npm install can stand in for a VolView worktree if one is not available.", + "Recording it here is what lets a machine other than the author's (or CI)", + "reproduce the pairing at all — VOLVIEW_ROOT branch names cannot travel." + ] + } +} diff --git a/e2e/compat.setup.ts b/e2e/compat.setup.ts new file mode 100644 index 0000000..1fcb2d9 --- /dev/null +++ b/e2e/compat.setup.ts @@ -0,0 +1,67 @@ +import { request as playwrightRequest, FullConfig } from '@playwright/test'; +import { healthCheck, verifyDeployedHeads, fetchDeployReceipt } from './helpers/stack'; +import { provisionCompat } from './helpers/compat-provision'; +import { readCompatState, writeCompatState, COMPAT_STATE_PATH } from './helpers/compat-state'; + +// Phase-switched global setup for the browser harness. +// +// COMPAT_PHASE=capture — against the MAIN deploy (E2E_EXPECT_GIRDER_SHA and +// E2E_EXPECT_VOLVIEW_SHA carry the pinned pair): +// provision the run folder ONCE and write +// .compat-state.json. +// COMPAT_PHASE=verify — against THIS worktree: restore captured sessions. +// COMPAT_PHASE=current — against THIS worktree: exercise fresh current +// lifecycles and jobs in otherwise untouched folders. +export default async function compatSetup(_config: FullConfig): Promise { + const phase = process.env.COMPAT_PHASE; + if (phase !== 'capture' && phase !== 'verify' && phase !== 'current') { + throw new Error( + `[compat] COMPAT_PHASE must be 'capture', 'verify', or 'current' (got '${phase ?? ''}'). ` + + 'Run via e2e/scripts/compat.sh.' + ); + } + + const request = await playwrightRequest.newContext({ ignoreHTTPSErrors: true }); + try { + await healthCheck(request); + await verifyDeployedHeads(request); + + if (phase === 'capture') { + if (readCompatState()) { + throw new Error( + `[compat] ${COMPAT_STATE_PATH} already exists — a previous capture was not ` + + 'verified/torn down. Run the verify phase (or delete the state file and its ' + + 'run folder) first.' + ); + } + const receipt = await fetchDeployReceipt(request); + const state = await provisionCompat(request, { + sourceGirderSha: receipt.girderSha || '', + sourceVolviewSha: receipt.volviewSha || '', + }); + writeCompatState(state); + // eslint-disable-next-line no-console + console.log( + `[compat] capture provisioned: root ${state.runRootFolderId} ` + + `with ${Object.keys(state.fixtures).length} isolated fixture folders ` + + `against source girder ${state.sourceGirderSha.slice(0, 9)}` + ); + } else { + const state = readCompatState(); + if (!state?.provisioned) { + throw new Error('[compat] no .compat-state.json — run the capture phase first.'); + } + if (phase === 'verify' && state.gestures.length === 0) { + throw new Error('[compat] capture recorded no gestures — nothing to verify.'); + } + // eslint-disable-next-line no-console + console.log( + `[compat] ${phase === 'verify' ? 'verifying' : 'running current scenarios with'} ` + + `${state.gestures.length} captured gesture(s) from source girder ` + + `${state.sourceGirderSha.slice(0, 9)}` + ); + } + } finally { + await request.dispose(); + } +} diff --git a/e2e/compat.teardown.ts b/e2e/compat.teardown.ts new file mode 100644 index 0000000..05eea85 --- /dev/null +++ b/e2e/compat.teardown.ts @@ -0,0 +1,26 @@ +import { request as playwrightRequest, FullConfig } from '@playwright/test'; +import { readCompatState, clearCompatState } from './helpers/compat-state'; +import { teardownCompat } from './helpers/compat-provision'; + +// The orchestrator chooses the final Playwright invocation by setting +// COMPAT_CLEANUP=1. Earlier phases leave the run root and state in place. +export default async function compatTeardown(_config: FullConfig): Promise { + if (process.env.COMPAT_CLEANUP !== '1') return; + if (process.env.COMPAT_KEEP === '1') { + // eslint-disable-next-line no-console + console.log('[compat] COMPAT_KEEP=1 — keeping run folder and state for iteration.'); + return; + } + + const state = readCompatState(); + if (!state) return; + if (state.provisioned) { + const request = await playwrightRequest.newContext({ ignoreHTTPSErrors: true }); + try { + await teardownCompat(request, state); + } finally { + await request.dispose(); + } + } + clearCompatState(); +} diff --git a/e2e/fixtures/dicom.large_image_config.yaml b/e2e/fixtures/dicom.large_image_config.yaml new file mode 100644 index 0000000..96b3ee6 --- /dev/null +++ b/e2e/fixtures/dicom.large_image_config.yaml @@ -0,0 +1,44 @@ +# Item-list config for the compat suite's small DICOM folder: one row per +# series, filterable on the metadata columns. The compat gestures launch via the +# folder-header "Open Checked in VolView" button (checkboxes), NOT by clicking a +# row, so this list intentionally omits `navigate: {type: open}` — that path in +# large_image throws on the `.large_image_config.yaml` item itself (which has no +# openable app), leaving the item list stuck on its loading spinner. +# +# Grouping keys resolve against item metadata as `meta.dicom.*`, which +# girder_volview populates when `seed.py seed-small` uploads the slices. + +defaultItemList: seriesList + +namedItemLists: + seriesList: + # `flatten: only` activates the recurse+group fetch path; without it + # large_image leaves a flat folder ungrouped (one row per slice). + layout: + flatten: only + group: + keys: + - dicom.SeriesInstanceUID + counts: + _id: _count.slicescount + defaultSort: + - type: metadata + value: dicom.PatientID + dir: up + columns: + - type: metadata + value: dicom.PatientID + title: Patient ID + format: text + - type: metadata + value: dicom.Modality + title: Modality + - type: metadata + value: dicom.SeriesDescription + title: Series Description + - type: metadata + value: _count.slicescount + title: Slices + format: count + - type: record + value: controls diff --git a/e2e/helpers/annotations.ts b/e2e/helpers/annotations.ts new file mode 100644 index 0000000..5854081 --- /dev/null +++ b/e2e/helpers/annotations.ts @@ -0,0 +1,154 @@ +import { Page, expect } from '@playwright/test'; +import { openModuleTab } from './volview'; +import { RulerRecord } from './compat-state'; + +// Content creation + readback inside VolView, selector-compatible with both the +// main-era client (capture) and the branch client (verify) — grounded on main's +// ControlsStripTools / AnnotationsModule / SegmentGroupControls / +// PatientStudyVolumeBrowser markup, which the branch retains. + +async function first2DCanvasBox(page: Page) { + const canvas = page + .locator('div[data-testid~="vtk-two-view"] canvas, div[data-testid~="vtk-cine-view"] canvas') + .first(); + await expect(canvas, 'no 2D view canvas').toBeVisible(); + const box = await canvas.boundingBox(); + if (!box || box.width < 50 || box.height < 50) { + throw new Error(`2D view canvas has no usable size: ${JSON.stringify(box)}`); + } + return box; +} + +async function activateTool(page: Page, icon: 'mdi-ruler' | 'mdi-brush'): Promise { + const button = page.locator(`button:has(i.${icon})`).first(); + await expect(button, `no ${icon} tool button`).toBeVisible(); + await button.click(); +} + +// Two clicks at center±40px on the first 2D view — the same gesture VolView's +// own wdio suite uses to place a ruler. +export async function placeRuler(page: Page): Promise { + await activateTool(page, 'mdi-ruler'); + const box = await first2DCanvasBox(page); + const cx = box.x + box.width / 2; + const cy = box.y + box.height / 2; + await page.mouse.click(cx - 40, cy); + await page.waitForTimeout(300); + await page.mouse.click(cx + 40, cy); + await page.waitForTimeout(500); +} + +const openAnnotationsTab = async (page: Page, tabText: 'Measurements' | 'Segment Groups') => { + await openModuleTab(page, 'Annotations'); + const tab = page.locator('.v-tab', { hasText: tabText }).first(); + await expect(tab, `no "${tabText}" tab in the Annotations module`).toBeVisible(); + await tab.click(); +}; + +// Ruler rows in the Measurements list; the rendered length ("40.00mm") comes +// from world coordinates in the session, so restores must reproduce it exactly. +export async function readRulerMeasurements(page: Page): Promise { + await openAnnotationsTab(page, 'Measurements'); + const rows = page.locator('.v-list-item:has(i.tool-icon.mdi-ruler)'); + await expect(rows.first(), 'no ruler row in the Measurements list').toBeVisible(); + const texts = await rows.allTextContents(); + return texts + .map((t) => t.match(/\d+\.\d{2}\s*mm/)?.[0]?.replace(/\s+/, '')) + .filter((t): t is string => !!t) + .map((lengthText) => ({ lengthText })); +} + +// Paint a few strokes on the first 2D view. Activating paint (and stroking) +// auto-creates a segment group for the current image when none exists. +export async function paintStrokes(page: Page): Promise { + await activateTool(page, 'mdi-brush'); + await page.waitForTimeout(500); + const box = await first2DCanvasBox(page); + const cx = box.x + box.width / 2; + const cy = box.y + box.height / 2; + for (const [dx, dy] of [ + [-30, -20], + [-10, 15], + ]) { + await page.mouse.move(cx + dx, cy + dy); + await page.mouse.down(); + await page.mouse.move(cx + dx + 40, cy + dy + 10, { steps: 8 }); + await page.mouse.up(); + await page.waitForTimeout(300); + } +} + +export async function readSegmentGroupNames(page: Page): Promise { + await openAnnotationsTab(page, 'Segment Groups'); + const names = page.locator('.segment-group-list .group-name'); + await expect(names.first(), 'no segment group listed').toBeVisible(); + return (await names.allTextContents()).map((t) => t.trim()).filter(Boolean); +} + +const dicomVolumeCard = (page: Page, seriesDescription: string) => + page.locator('.v-card', { has: page.locator('.series-desc') }).filter({ hasText: seriesDescription }).first(); + +export async function readDatasetNames(page: Page): Promise { + await openModuleTab(page, 'Data'); + // DICOM volumes render series-desc cards; non-DICOM images render name rows. + const dicomNames = await page.locator('.series-desc .text-ellipsis').allTextContents(); + const imageNames = await page + .locator('.text-body-2.font-weight-bold.text-no-wrap.text-truncate') + .allTextContents(); + return [...dicomNames, ...imageNames].map((t) => t.trim()).filter(Boolean); +} + +// Make the volume with this series description the PRIMARY selection (layer +// targets attach to the primary). +export async function selectPrimaryVolume(page: Page, seriesDescription: string): Promise { + await openModuleTab(page, 'Data'); + const card = dicomVolumeCard(page, seriesDescription); + await expect(card, `no volume card for "${seriesDescription}"`).toBeVisible(); + await card.click(); + await page.waitForTimeout(1_000); +} + +async function openDatasetMenu(page: Page, seriesDescription: string) { + await openModuleTab(page, 'Data'); + const card = dicomVolumeCard(page, seriesDescription); + await expect(card, `no volume card for "${seriesDescription}"`).toBeVisible(); + await card.locator('[data-testid="dataset-menu-button"]').click(); + return page.locator('.v-overlay-container [data-testid="dataset-menu-layer-item"]').first(); +} + +// Ensure the volume is layered onto the primary. VolView auto-layers PET over +// CT when a whole CT+PET study loads, so "already layered" is success, not an +// error. +export async function addLayer(page: Page, seriesDescription: string): Promise { + const layerItem = await openDatasetMenu(page, seriesDescription); + await expect(layerItem, `no layer menu item for "${seriesDescription}"`).toBeVisible(); + const initial = (await layerItem.textContent().catch(() => '')) || ''; + if (initial.includes('Remove as layer')) { + await page.keyboard.press('Escape'); + return; + } + await layerItem.click(); + // Layer load is async; poll the menu until it reads "Remove as layer". + await expect + .poll( + async () => { + await page.keyboard.press('Escape'); + await page.waitForTimeout(500); + const item = await openDatasetMenu(page, seriesDescription); + const text = (await item.textContent().catch(() => '')) || ''; + await page.keyboard.press('Escape'); + return text; + }, + { timeout: 60_000, message: 'layer never finished loading' } + ) + .toContain('Remove as layer'); + await page.keyboard.press('Escape'); +} + +export async function isLayered(page: Page, seriesDescription: string): Promise { + const layerItem = await openDatasetMenu(page, seriesDescription); + const text = (await layerItem.textContent().catch(() => '')) || ''; + await page.keyboard.press('Escape'); + await page.waitForTimeout(300); + return text.includes('Remove as layer'); +} diff --git a/e2e/helpers/compat-provision.ts b/e2e/helpers/compat-provision.ts new file mode 100644 index 0000000..62a4b32 --- /dev/null +++ b/e2e/helpers/compat-provision.ts @@ -0,0 +1,184 @@ +import { execFileSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import { APIRequestContext } from '@playwright/test'; +import { CONFIG, apiUrl } from './config'; +import { readJson } from './http'; +import { makeNrrd } from './nrrd'; +import { authenticate, createFolderUnder, uploadFile, deleteFolder } from './provision'; +import { CompatState, FixtureFolder, FixtureId } from './compat-state'; + +// Provision one run root containing an isolated folder for every scenario. +// NRRD fixtures are generated in memory; grouped fixtures plain-upload cached +// IDC DICOM slices and an item-list config that groups on meta.dicom.*. + +const REPO_ROOT = path.resolve(__dirname, '..', '..'); +const SEED_CLI = path.resolve(__dirname, '..', 'seed', 'seed.py'); +const DICOM_LI_CONFIG = path.resolve(__dirname, '..', 'fixtures', 'dicom.large_image_config.yaml'); + +function seedSmallDicom(folderId: string): void { + // eslint-disable-next-line no-console + console.log(`[compat] seeding small DICOM tier into folder ${folderId} (uv run seed.py)`); + execFileSync('uv', ['run', SEED_CLI, 'seed-small', '--folder-id', folderId, '--slices', '12'], { + cwd: REPO_ROOT, + stdio: 'inherit', + env: { + ...process.env, + GIRDER_URL: CONFIG.baseURL, + DSA_ADMIN_USER: CONFIG.user, + DSA_ADMIN_PASS: CONFIG.pass, + }, + }); +} + +async function listItems( + request: APIRequestContext, + token: string, + folderId: string +): Promise> { + const res = await request.get(apiUrl(`/item?folderId=${folderId}&limit=1000`), { + headers: { 'Girder-Token': token }, + }); + return readJson(res, `list items of ${folderId}`); +} + +// Probe for the optional devkit tier: the trial folder of the "VolView Devkit" +// collection, when the full devkit has been seeded. +export async function findDevkitTrialFolder( + request: APIRequestContext, + token: string +): Promise { + const collRes = await request.get( + apiUrl(`/collection?text=${encodeURIComponent('VolView Devkit')}&limit=10`), + { headers: { 'Girder-Token': token } } + ); + const collections: Array<{ _id: string; name: string }> = await collRes.json(); + const devkit = collections.find?.((c) => c.name === 'VolView Devkit'); + if (!devkit) return undefined; + const folderRes = await request.get( + apiUrl(`/folder?parentType=collection&parentId=${devkit._id}&name=trial`), + { headers: { 'Girder-Token': token } } + ); + const folders: Array<{ _id: string }> = await folderRes.json(); + return folders?.[0]?._id; +} + +export async function provisionCompat( + request: APIRequestContext, + deployed: { sourceGirderSha: string; sourceVolviewSha: string } +): Promise { + const { token, userId } = await authenticate(request); + + const runId = `${Date.now()}-${Math.floor(Math.random() * 1e4)}`; + const runRootFolderId = await createFolderUnder( + request, + token, + 'user', + userId, + `girder-volview-compat-${runId}` + ); + const fixtures = {} as Record; + + async function provision(): Promise { + async function nrrdFixture(id: FixtureId, count: 1 | 2): Promise { + const folderId = await createFolderUnder(request, token, 'folder', runRootFolderId, id); + const uploaded = []; + for (let index = 0; index < count; index += 1) { + uploaded.push( + await uploadFile( + request, + token, + folderId, + `synthetic-${index + 1}.nrrd`, + makeNrrd({ variant: index }) + ) + ); + } + fixtures[id] = { + folderId, + itemIds: uploaded.map((item) => item.itemId), + itemNames: uploaded.map((item) => item.itemName), + }; + } + + async function dicomFixture(id: FixtureId): Promise { + const folderId = await createFolderUnder(request, token, 'folder', runRootFolderId, id); + // large_image's grouped recursive endpoint assumes a flattened folder has + // at least one descendant. Keep the config and resulting session items at + // the scenario root, and put the source slices in a public child folder. + // This also matches the hierarchy shape that flatten/group is meant for. + const dataFolderId = await createFolderUnder(request, token, 'folder', folderId, 'dicom'); + seedSmallDicom(dataFolderId); + await uploadFile( + request, + token, + folderId, + '.large_image_config.yaml', + fs.readFileSync(DICOM_LI_CONFIG) + ); + const dicomItems = await listItems(request, token, dataFolderId); + const images = dicomItems.filter((item) => item.name.endsWith('.dcm')); + if (images.length === 0) throw new Error(`[compat] fixture '${id}' contains no DICOM items`); + fixtures[id] = { + folderId, + itemIds: images.map((item) => item._id), + itemNames: images.map((item) => item.name), + }; + } + + await nrrdFixture('single-item', 1); + await nrrdFixture('checked-nrrd', 2); + await dicomFixture('filtered-dicom'); + await dicomFixture('study-layered'); + + await nrrdFixture('lifecycle-single', 1); + await nrrdFixture('lifecycle-checked', 2); + await dicomFixture('lifecycle-filter'); + await nrrdFixture('lifecycle-restart', 2); + await nrrdFixture('lifecycle-bare', 2); + await nrrdFixture('lifecycle-older', 2); + await nrrdFixture('lifecycle-session-row', 2); + await nrrdFixture('lifecycle-url-contract', 2); + await nrrdFixture('jobs-comeback', 1); + await nrrdFixture('jobs-live', 1); + await nrrdFixture('jobs-staged', 1); + await nrrdFixture('jobs-failure', 1); + + const devkitTrialFolderId = await findDevkitTrialFolder(request, token); + + return { + createdAt: new Date().toISOString(), + sourceGirderSha: deployed.sourceGirderSha, + sourceVolviewSha: deployed.sourceVolviewSha, + runRootFolderId, + fixtures, + token, + provisioned: true, + dicomSeeded: true, + devkitTrialFolderId, + gestures: [], + }; + } + + try { + return await provision(); + } catch (error) { + if (!(await deleteFolder(request, token, runRootFolderId))) { + const message = error instanceof Error ? error.message : String(error); + throw new Error( + `[compat] provisioning failed and could not delete run root ${runRootFolderId}: ${message}` + ); + } + throw error; + } +} + +export async function teardownCompat(request: APIRequestContext, state: CompatState): Promise { + let token = state.token; + try { + token = (await authenticate(request)).token; + } catch { + /* use the stored token */ + } + await deleteFolder(request, token, state.runRootFolderId); +} diff --git a/e2e/helpers/compat-state.ts b/e2e/helpers/compat-state.ts new file mode 100644 index 0000000..4c0b485 --- /dev/null +++ b/e2e/helpers/compat-state.ts @@ -0,0 +1,129 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +// State shared by the harness's Playwright invocations: baseline capture, +// current verification, and current-only behavior. Capture writes it and the +// orchestrator's final phase deletes it unless COMPAT_KEEP=1. + +export type RulerRecord = { + // The rendered measurement, e.g. "40.00mm" — world coordinates live in the + // session zip, so a faithful restore reproduces this text exactly. + lengthText: string; +}; + +// Semantic summary of a session zip's manifest.json — migration-tolerant +// (counts and presence, not raw JSON equality). +export type ZipSummary = { + rulerCount: number; + segmentGroupCount: number; + // Size of the largest segment-group archive entry: painted voxels make it + // decidedly non-trivial, an empty labelmap does not. + segmentGroupDataBytes: number; + hasLayers: boolean; + version?: string; +}; + +export type GestureId = + | 'checked-nrrd' + | 'filtered-dicom' + | 'study-layered' + | 'single-item' + | 'devkit-study'; + +export type FixtureId = + | Exclude + | 'lifecycle-single' + | 'lifecycle-checked' + | 'lifecycle-filter' + | 'lifecycle-restart' + | 'lifecycle-bare' + | 'lifecycle-older' + | 'lifecycle-session-row' + | 'lifecycle-url-contract' + | 'jobs-comeback' + | 'jobs-live' + | 'jobs-staged' + | 'jobs-failure'; + +export type FixtureFolder = { + folderId: string; + itemIds: string[]; + itemNames: string[]; +}; + +export type LaunchDescriptor = + | { via: 'checked-items'; itemIds: string[] } + // Grouped rows matched by the conjunction of cell texts, optionally after + // narrowing with the filter box. + | { via: 'checked-rows'; rows: string[][]; filterText?: string } + | { via: 'item-page'; itemId: string } + | { via: 'row-nav'; rowTexts: string[] }; + +export type CapturedGesture = { + id: GestureId; + folderId: string; + launch: LaunchDescriptor; + // Folder-scoped saves mint a session item (absent for single-item saves, + // where the zip lands inside the launched item). + sessionItemId?: string; + sessionItemName?: string; + expected: { + datasetNames: string[]; + rulers: RulerRecord[]; + segmentGroupNames: string[]; + petLayer: boolean; + zip: ZipSummary; + }; +}; + +export type CompatState = { + createdAt: string; + sourceGirderSha: string; + sourceVolviewSha: string; + runRootFolderId: string; + // Every scenario owns its folder. Saves and checkbox state from one test can + // therefore never alter another test's launch semantics. + fixtures: Record; + token: string; + provisioned: boolean; + dicomSeeded: boolean; + devkitTrialFolderId?: string; + gestures: CapturedGesture[]; +}; + +export const COMPAT_STATE_PATH = path.resolve(__dirname, '..', '.compat-state.json'); + +export function writeCompatState(state: CompatState): void { + fs.writeFileSync(COMPAT_STATE_PATH, JSON.stringify(state, null, 2), 'utf8'); +} + +export function readCompatState(): CompatState | undefined { + try { + return JSON.parse(fs.readFileSync(COMPAT_STATE_PATH, 'utf8')) as CompatState; + } catch { + return undefined; + } +} + +export function clearCompatState(): void { + try { + fs.unlinkSync(COMPAT_STATE_PATH); + } catch { + /* already gone */ + } +} + +export function requireFixture(state: CompatState, id: FixtureId): FixtureFolder { + const fixture = state.fixtures[id]; + if (!fixture) throw new Error(`[compat] fixture '${id}' was not provisioned`); + return fixture; +} + +// Capture specs append gestures one test at a time; the harness uses one worker, +// so read-modify-write keeps the file the source of truth across the phase. +export function appendGesture(gesture: CapturedGesture): void { + const state = readCompatState(); + if (!state) throw new Error('[compat] no state file — did capture setup run?'); + state.gestures = [...state.gestures.filter((g) => g.id !== gesture.id), gesture]; + writeCompatState(state); +} diff --git a/e2e/helpers/config.ts b/e2e/helpers/config.ts new file mode 100644 index 0000000..7f039a6 --- /dev/null +++ b/e2e/helpers/config.ts @@ -0,0 +1,13 @@ +// Deployment config — a committed literal (no environment variables). +export const CONFIG = { + // girder origin serving the VolView dist (/static/built/plugins/volview/). + baseURL: 'http://localhost:8080', + // girder API mount (e.g. 'girder/api/v1' behind a prefix). + apiRoot: 'api/v1', + // girder login; the token is also planted as the `girderToken` cookie. + user: 'admin', + pass: 'password', +} as const; + +// Absolute girder REST URL for an api path (path must start with '/'). +export const apiUrl = (path: string) => `${CONFIG.baseURL}/${CONFIG.apiRoot}${path}`; diff --git a/e2e/helpers/girder-ui.ts b/e2e/helpers/girder-ui.ts new file mode 100644 index 0000000..d874bd5 --- /dev/null +++ b/e2e/helpers/girder-ui.ts @@ -0,0 +1,166 @@ +import { Page, expect } from '@playwright/test'; +import { CONFIG } from './config'; +import { isManifestGet, requireManifestJson } from './manifest'; + +// Drivers for the real Girder web UI: the large_image item list's filter box +// and checkboxes, and the plugin's Open-in-VolView affordances. All browser +// scenarios launch through these product paths, so open.js always builds the +// URL under test. + +// A VolView popup plus the valid manifest its boot fetched. +export type VolViewLaunch = { popup: Page; manifest: Promise }; + +async function watchManifest(popup: Page): Promise { + const response = await popup.waitForResponse(isManifestGet, { timeout: 90_000 }); + return requireManifestJson(response); +} + +async function toLaunch(popupPromise: Promise): Promise { + const popup = await popupPromise; + // Attach BEFORE the app boots far enough to fetch the manifest — the popup + // event fires at window creation, megabytes of app JS before any fetch. + const manifest = watchManifest(popup); + await popup.waitForLoadState('domcontentloaded'); + return { popup, manifest }; +} + +// Log the girder WEB CLIENT in through its UI. Planting the girderToken cookie +// authenticates VolView's cookie-accepting routes, but girder rejects a +// cookie-only token on state-changing requests (CSRF): main's folder-open path +// fires a metadata PUT before window.open, so without a real login that PUT +// 401s and the popup never opens. A UI login sets the client's currentToken, so +// its restRequest sends the Girder-Token header on writes. +export async function loginViaUI(page: Page): Promise { + await page.goto(`${CONFIG.baseURL}/#`, { waitUntil: 'domcontentloaded' }); + // DSA's "Log In" is an with no href, so it is NOT a link-role element — + // match it by tag + text. Absent → already authenticated. + const loginLink = page.locator('a', { hasText: /log ?in/i }).first(); + await page.waitForLoadState('networkidle').catch(() => undefined); + if (!(await loginLink.isVisible({ timeout: 10_000 }).catch(() => false))) { + return; + } + await loginLink.click(); + await page.locator('#g-login').fill(CONFIG.user); + await page.locator('#g-password').fill(CONFIG.pass); + await page.locator('#g-login-button').click(); + await expect( + page.locator('a', { hasText: /log ?in/i }).first(), + 'girder UI login did not complete (Log In link still present)' + ).toBeHidden({ timeout: 30_000 }); +} + +// Navigate to a folder. When the girder SPA is already loaded (e.g. right after +// loginViaUI), change the hash IN-APP rather than page.goto — a full reload +// drops the client's in-memory auth token, and girder keeps no header-usable +// cookie, so the next write (main's pre-open metadata PUT) would 401. +export async function gotoFolder(page: Page, folderId: string): Promise { + if (page.url().startsWith(CONFIG.baseURL)) { + await page.evaluate((id) => { + window.location.hash = `#folder/${id}`; + }, folderId); + } else { + await page.goto(`${CONFIG.baseURL}/#folder/${folderId}`, { waitUntil: 'domcontentloaded' }); + } + await expect(page.locator('li.g-item-list-entry').first()).toBeVisible({ timeout: 30_000 }); +} + +export async function checkRowByItemId(page: Page, itemId: string): Promise { + const row = page.locator(`li.g-item-list-entry:has(a[href="#item/${itemId}"])`); + await expect(row, `no item-list row for item ${itemId}`).toBeVisible(); + await row.locator('input.g-list-checkbox').check(); +} + +// Clear every checked row. gotoFolder changes the hash IN-APP, so checkbox +// state survives navigation — a "bare folder-open" right after a checked +// gesture would otherwise still carry the earlier selection. +export async function uncheckAllRows(page: Page): Promise { + const checked = () => page.locator('input.g-list-checkbox:checked'); + // Bounded: each uncheck removes one from the set. + for (let guard = 0; guard < 50 && (await checked().count()) > 0; guard += 1) { + await checked().first().uncheck(); + } + await expect(checked(), 'rows remained checked').toHaveCount(0); +} + +// Grouped item lists render metadata columns as cell text; match a row by the +// conjunction of distinctive cell values (e.g. PatientID + SeriesDescription). +export function rowByTexts(page: Page, texts: string[]) { + return texts.reduce( + (rows, text) => rows.filter({ hasText: text }), + page.locator('li.g-item-list-entry') + ); +} + +export async function checkRowByTexts(page: Page, texts: string[]): Promise { + const row = rowByTexts(page, texts).first(); + await expect(row, `no item-list row containing ${JSON.stringify(texts)}`).toBeVisible(); + await row.locator('input.g-list-checkbox').check(); +} + +// large_image's item-list filter box narrows the listing server-side. +export async function fillFilterBox(page: Page, text: string): Promise { + const box = page.locator('input.li-item-list-filter-input').first(); + await expect(box, 'no large_image filter box — is .large_image_config.yaml applied?').toBeVisible(); + await box.fill(text); + await box.press('Enter'); +} + +// Presence/absence of rows by cell-text conjunction. Count assertions are +// deliberately avoided: the .large_image_config.yaml item renders as its own +// (ungrouped) row, so absolute row counts are layout-dependent. +export async function expectRow(page: Page, texts: string[]): Promise { + await expect(rowByTexts(page, texts).first(), `expected a row containing ${JSON.stringify(texts)}`) + .toBeVisible({ timeout: 30_000 }); +} + +export async function expectNoRow(page: Page, texts: string[]): Promise { + await expect + .poll(async () => rowByTexts(page, texts).count(), { + timeout: 30_000, + message: `expected NO row containing ${JSON.stringify(texts)}`, + }) + .toBe(0); +} + +// Click the hierarchy header's Open-in-VolView button and hand back the popup. +// A "Will open newest VolView session" confirm modal may interpose (when a +// session item is among the checked rows). +export async function openInVolView(page: Page): Promise { + const button = page.locator('.open-in-volview'); + await expect(button, 'Open-in-VolView button not visible').toBeVisible(); + const popupPromise = page.waitForEvent('popup', { timeout: 60_000 }); + await button.click(); + const modal = page.locator('.modal-content:has-text("Will open newest VolView session")'); + if (await modal.isVisible({ timeout: 2_000 }).catch(() => false)) { + await page.locator('#g-confirm-button').click(); + } + return toLaunch(popupPromise); +} + +// Open from the item page (the single-item gesture). +export async function openFromItemPage(page: Page, itemId: string): Promise { + await page.goto(`${CONFIG.baseURL}/#item/${itemId}`, { waitUntil: 'domcontentloaded' }); + const button = page.locator('.open-in-volview'); + await expect(button, 'no Open-in-VolView on the item page').toBeVisible(); + const popupPromise = page.waitForEvent('popup', { timeout: 60_000 }); + await button.click(); + return toLaunch(popupPromise); +} + +// Drill through rows whose navigate is another item list (devkit patient → +// study); the LAST row's navigate opens VolView, so the final click yields the +// popup. +export async function drillRowNav(page: Page, rowTexts: string[]): Promise { + for (const text of rowTexts.slice(0, -1)) { + const row = rowByTexts(page, [text]).first(); + await expect(row, `no drill-down row containing "${text}"`).toBeVisible(); + await row.locator('a.g-item-list-link').first().click(); + await expect(page.locator('li.g-item-list-entry').first()).toBeVisible({ timeout: 30_000 }); + } + const lastText = rowTexts[rowTexts.length - 1]; + const last = rowByTexts(page, [lastText]).first(); + await expect(last, `no final row containing "${lastText}"`).toBeVisible(); + const popupPromise = page.waitForEvent('popup', { timeout: 60_000 }); + await last.locator('a.g-item-list-link').first().click(); + return toLaunch(popupPromise); +} diff --git a/e2e/helpers/girder.ts b/e2e/helpers/girder.ts new file mode 100644 index 0000000..b642aca --- /dev/null +++ b/e2e/helpers/girder.ts @@ -0,0 +1,107 @@ +import { APIRequestContext, BrowserContext, expect } from '@playwright/test'; +import { CONFIG, apiUrl } from './config'; +import { + CompatState, + FixtureId, + readCompatState, + requireFixture, +} from './compat-state'; + +export { CONFIG }; + +export type Girder = { + token: string; + folderId: string; + itemId: string; + itemName: string; + itemIds: string[]; + itemNames: string[]; +}; + +const api = apiUrl; + +export async function plantCookie(context: BrowserContext, token: string) { + const { hostname } = new URL(CONFIG.baseURL); + await context.addCookies([ + { name: 'girderToken', value: token, domain: hostname, path: '/' }, + ]); +} + +export function requireHarnessState(): CompatState { + const state = readCompatState(); + if (!state?.provisioned) { + throw new Error('[e2e] no harness state — run through e2e/scripts/compat.sh'); + } + return state; +} + +export async function setupFixture(context: BrowserContext, id: FixtureId): Promise { + const state = requireHarnessState(); + const fixture = requireFixture(state, id); + await plantCookie(context, state.token); + return { + token: state.token, + folderId: fixture.folderId, + itemId: fixture.itemIds[0], + itemName: fixture.itemNames[0], + itemIds: fixture.itemIds, + itemNames: fixture.itemNames, + }; +} + +// The session.volview.zip items currently in a folder. countSessionItems proves +// a save created a NEW session item; the compat capture diffs the listing to +// discover which item a save minted (main's save response carries no resumeUrl). +export async function listSessionItems( + request: APIRequestContext, + token: string, + folderId: string +): Promise> { + const res = await request.get(api(`/item?folderId=${folderId}&limit=1000`), { + headers: { 'Girder-Token': token }, + }); + const items: Array<{ _id: string; name: string }> = await res.json(); + // Substring, not endsWith: girder dedupes colliding item names by appending + // " (1)", and the backend's isSessionItem treats those as sessions too. + return items.filter((it) => it.name.includes('.volview.zip')); +} + +export async function countSessionItems(request: APIRequestContext, g: Girder): Promise { + return (await listSessionItems(request, g.token, g.folderId)).length; +} + +// The id of an item's first file. Two saves collide on file NAME (girder +// dedupes the item name, "session.volview.zip (1)", while the file inside keeps +// the original), so the file id is what distinguishes one save from another in +// a manifest — resources carry it in their minted /file//proxiable URL. +export async function firstFileId( + request: APIRequestContext, + token: string, + itemId: string +): Promise { + const res = await request.get(api(`/item/${itemId}/files?limit=1`), { + headers: { 'Girder-Token': token }, + }); + expect(res.ok(), `GET /item/${itemId}/files returned HTTP ${res.status()}`).toBeTruthy(); + const files: Array<{ _id: string }> = await res.json(); + expect(files?.[0]?._id, `item ${itemId} has no files`).toBeTruthy(); + return files[0]._id; +} + +export const resourceUrls = (json: any): string[] => + Array.isArray(json?.resources) ? json.resources.map((r: any) => r?.url).filter(Boolean) : []; + +// Fetch a manifest by the `urls=` leg a launched tab is carrying. Used to +// inspect WHICH resources a launch resolved to without racing the tab's own +// in-flight request (a popup can finish loading before an interceptor attaches). +export async function fetchManifest( + request: APIRequestContext, + token: string, + urls: string +): Promise { + const res = await request.get(`${CONFIG.baseURL}${urls}`, { + headers: { 'Girder-Token': token }, + }); + expect(res.ok(), `manifest ${urls} returned HTTP ${res.status()}`).toBeTruthy(); + return res.json(); +} diff --git a/e2e/helpers/http.ts b/e2e/helpers/http.ts new file mode 100644 index 0000000..cdd2c8e --- /dev/null +++ b/e2e/helpers/http.ts @@ -0,0 +1,16 @@ +import { APIResponse } from '@playwright/test'; + +// Parse a girder REST response as JSON, failing with a bounded excerpt of the +// body — the one HTTP-error convention for the whole e2e helper suite. +export async function readJson(res: APIResponse, ctx: string): Promise { + const status = res.status(); + const text = await res.text(); + if (status >= 300) { + throw new Error(`[e2e] ${ctx} failed: HTTP ${status} ${text.slice(0, 400)}`); + } + try { + return JSON.parse(text); + } catch { + throw new Error(`[e2e] ${ctx}: non-JSON response: ${text.slice(0, 200)}`); + } +} diff --git a/e2e/helpers/jobs.ts b/e2e/helpers/jobs.ts new file mode 100644 index 0000000..a3c90f4 --- /dev/null +++ b/e2e/helpers/jobs.ts @@ -0,0 +1,109 @@ +import { APIRequestContext } from '@playwright/test'; +import { CONFIG, apiUrl } from './config'; +import { readJson } from './http'; + +// Processing-job REST helpers — submit a task and poll it to a terminal state, +// mirroring the requests the browser client mints. + +// The proxiable URI the backend resolves back to a Girder file id (and re-checks +// READ ACL): origin-relative /{apiRoot}/file//proxiable/. +function proxiableUri(fileId: string, name: string): string { + return `/${CONFIG.apiRoot}/file/${fileId}/proxiable/${encodeURIComponent(name)}`; +} + +// The proxiable URIs for every file in an item (one per slice for a series, one +// for a single volume) — the CLI's input binding. +export async function itemInputUris( + request: APIRequestContext, + token: string, + itemId: string +): Promise { + const res = await request.get(apiUrl(`/item/${itemId}/files?limit=1000`), { + headers: { 'Girder-Token': token }, + }); + const files: Array<{ _id: string; name: string }> = await readJson(res, `list files ${itemId}`); + if (files.length === 0) throw new Error(`[e2e] item ${itemId} has no files to bind as input`); + return files.map((f) => proxiableUri(f._id, f.name)); +} + +// Find a registered processing task by title prefix (e.g. 'Otsu'), scoped to the +// folder the run will target. Tasks come from the registered radiology CLI image. +export async function findTask( + request: APIRequestContext, + token: string, + folderId: string, + titlePrefix: string +): Promise<{ id: string; title: string }> { + const res = await request.get(apiUrl(`/folder/${folderId}/volview_processing/tasks`), { + headers: { 'Girder-Token': token }, + }); + const tasks: Array<{ id: string; title: string }> = await readJson(res, 'list tasks'); + const task = tasks.find((t) => (t.title || '').startsWith(titlePrefix)); + if (!task) { + throw new Error( + `[e2e] no "${titlePrefix}*" task registered — register the radiology CLI ` + + `image with slicer_cli_web; tasks=${JSON.stringify(tasks.map((t) => t.title))}` + ); + } + return task; +} + +// Submit a task run against a folder. `values` is the client-faithful body: the +// bound input + scalar params, NO output entries (the backend autofills them). +// Returns the created girder job id. +export async function runTask( + request: APIRequestContext, + token: string, + folderId: string, + taskId: string, + values: Record +): Promise { + const res = await request.post(apiUrl(`/folder/${folderId}/volview_processing/tasks/${taskId}/run`), { + headers: { 'Girder-Token': token, 'Content-Type': 'application/json' }, + data: { values }, + }); + const body = await readJson(res, `run task ${taskId}`); + const jobId = body?.jobId; + if (!jobId) throw new Error(`[e2e] task run returned no jobId: ${JSON.stringify(body).slice(0, 300)}`); + return jobId; +} + +const TERMINAL = new Set(['success', 'error', 'cancelled']); + +// Poll a job to a terminal state (success/error/cancelled) or throw on timeout. +export async function pollJob( + request: APIRequestContext, + token: string, + jobId: string, + timeoutMs = 240_000 +): Promise<{ state: string; [k: string]: unknown }> { + const deadline = Date.now() + timeoutMs; + let last: any; + while (Date.now() < deadline) { + const res = await request.get(apiUrl(`/volview_processing/jobs/${jobId}`), { + headers: { 'Girder-Token': token }, + }); + last = await readJson(res, `poll job ${jobId}`); + if (TERMINAL.has(last?.state)) return last; + await new Promise((r) => setTimeout(r, 2000)); + } + throw new Error(`[e2e] job ${jobId} did not reach a terminal state within ${timeoutMs}ms (last=${last?.state})`); +} + +// Submit an Otsu segmentation on the given item's image and poll to terminal. +// Returns { jobId, state }. +export async function submitOtsu( + request: APIRequestContext, + token: string, + folderId: string, + itemId: string +): Promise<{ jobId: string; state: string }> { + const uris = await itemInputUris(request, token, itemId); + const task = await findTask(request, token, folderId, 'Otsu'); + const jobId = await runTask(request, token, folderId, task.id, { + inputVolume: { type: 'image', uris }, + numberOfLevels: 3, + }); + const final = await pollJob(request, token, jobId); + return { jobId, state: final.state }; +} diff --git a/e2e/helpers/manifest.ts b/e2e/helpers/manifest.ts new file mode 100644 index 0000000..3bf1a30 --- /dev/null +++ b/e2e/helpers/manifest.ts @@ -0,0 +1,46 @@ +import { Page, expect, Response } from '@playwright/test'; +import { waitForVolViewReady } from './volview'; + +// Manifest interception shared by the lifecycle and compat suites: capture the +// GET /(item|folder)/:id/volview response a navigation triggers and classify +// whether it resumed a session (resources include a *.volview.zip) or loaded +// fresh raw images. + +export const isSessionManifest = (json: any) => + Array.isArray(json?.resources) && + json.resources.some((r: any) => typeof r?.name === 'string' && r.name.endsWith('.volview.zip')); + +export const resourceNames = (json: any): string[] => + Array.isArray(json?.resources) ? json.resources.map((r: any) => r?.name).filter(Boolean) : []; + +export const isManifestGet = (response: { request: () => { method: () => string }; url: () => string }) => + response.request().method() === 'GET' && + /\/(item|folder)\/[^/]+\/volview$/.test(new URL(response.url()).pathname); + +export async function requireManifestJson(response: Response): Promise { + const path = new URL(response.url()).pathname; + expect(response.ok(), `manifest ${path} returned HTTP ${response.status()}`).toBeTruthy(); + try { + return await response.json(); + } catch { + throw new Error(`manifest ${path} did not return valid JSON`); + } +} + +export async function captureManifest(page: Page, navigate: () => Promise): Promise { + const manifestResp = page.waitForResponse(isManifestGet, { timeout: 60_000 }); + await navigate(); + // Status BEFORE the readiness wait, on purpose: a failed manifest means the + // viewer never gets data, so waiting first turns a plain HTTP error into a + // 90s "viewer never became ready" timeout that names nothing. Checked here + // rather than in each caller so every launch and F5 gets it. + const manifest = await requireManifestJson(await manifestResp); + await waitForVolViewReady(page); + return manifest; +} + +export const gotoCapturingManifest = (page: Page, url: string) => + captureManifest(page, () => page.goto(url, { waitUntil: 'domcontentloaded' })); + +export const reloadCapturingManifest = (page: Page) => + captureManifest(page, () => page.reload({ waitUntil: 'domcontentloaded' })); diff --git a/e2e/helpers/nrrd.ts b/e2e/helpers/nrrd.ts new file mode 100644 index 0000000..50e17cb --- /dev/null +++ b/e2e/helpers/nrrd.ts @@ -0,0 +1,47 @@ +// In-process synthetic NRRD volume generator: a minimal valid NRRD (ASCII +// header + blank line + raw little-endian int16 voxels) that itk-wasm loads and +// renders. The voxels are a gradient so the render is visibly non-empty, with +// two `variant`s so the uploaded images look distinct. +export type NrrdOptions = { + size?: number; // cube edge length in voxels (default 16) + variant?: number; // 0 or 1 — flips the gradient so a.nrrd != b.nrrd visually +}; + +export function makeNrrd({ size = 16, variant = 0 }: NrrdOptions = {}): Buffer { + // Header — one field per line, terminated by a single blank line. Keep the + // geometry keys itk-wasm's NrrdImageIO expects for a well-formed LPS volume. + const header = + [ + 'NRRD0004', + '# Synthetic e2e test volume (generated in-process)', + 'type: short', + 'dimension: 3', + `sizes: ${size} ${size} ${size}`, + 'kinds: domain domain domain', + 'encoding: raw', + 'endian: little', + 'space: left-posterior-superior', + 'space directions: (1,0,0) (0,1,0) (0,0,1)', + 'space origin: (0,0,0)', + ].join('\n') + '\n\n'; // trailing blank line separates header from data + + const headerBuf = Buffer.from(header, 'ascii'); + const data = Buffer.alloc(size * size * size * 2); // int16 = 2 bytes/voxel + + // NRRD raw order: the first-listed axis varies fastest, so x is innermost. + const span = 3 * (size - 1) || 1; + let offset = 0; + for (let z = 0; z < size; z++) { + for (let y = 0; y < size; y++) { + for (let x = 0; x < size; x++) { + const g = (x + y + z) / span; // 0..1 corner-to-corner gradient + const norm = variant ? 1 - g : g; + const value = Math.round(norm * 3000); // fits comfortably in int16 + data.writeInt16LE(value, offset); + offset += 2; + } + } + } + + return Buffer.concat([headerBuf, data]); +} diff --git a/e2e/helpers/provision.ts b/e2e/helpers/provision.ts new file mode 100644 index 0000000..88c505f --- /dev/null +++ b/e2e/helpers/provision.ts @@ -0,0 +1,103 @@ +import { APIRequestContext } from '@playwright/test'; +import { CONFIG, apiUrl } from './config'; +import { readJson } from './http'; + +// Data provisioning over the girder REST API: authenticate, create a fresh +// public folder, and upload two synthetic NRRD images, so the gestures have +// loadable distinct picks without operator-created fixtures. + +// Basic-auth against girder; returns the session token + the user's _id. +export async function authenticate( + request: APIRequestContext +): Promise<{ token: string; userId: string }> { + const basic = Buffer.from(`${CONFIG.user}:${CONFIG.pass}`).toString('base64'); + const res = await request.get(apiUrl('/user/authentication'), { + headers: { Authorization: `Basic ${basic}` }, + }); + const body = await readJson(res, `auth (${CONFIG.user}@${CONFIG.baseURL})`); + const token = body?.authToken?.token; + const userId = body?.user?._id; + if (!token || !userId) { + throw new Error('[e2e] auth response missing authToken.token or user._id'); + } + return { token, userId }; +} + +// Create a public folder under any parent (user or folder). +export async function createFolderUnder( + request: APIRequestContext, + token: string, + parentType: 'user' | 'folder', + parentId: string, + name: string +): Promise { + const url = apiUrl( + `/folder?parentType=${parentType}&parentId=${parentId}` + + `&name=${encodeURIComponent(name)}&reuseExisting=false&public=true` + ); + const res = await request.post(url, { headers: { 'Girder-Token': token } }); + const folder = await readJson(res, `create folder ${name}`); + if (!folder?._id) throw new Error('[e2e] folder create returned no _id'); + // eslint-disable-next-line no-console + console.log(`[e2e] created folder ${folder._id} (${name})`); + return folder._id; +} + +// Upload one in-memory file to a folder via girder's two-step upload flow +// (init POST /file -> {_id}; then POST /file/chunk with the bytes as the raw +// request body). Modern girder REJECTS multipart on /file/chunk and reads the +// chunk from the body, with offset + uploadId in the query string (exactly what +// girder_client / the girder web client do). Girder auto-creates the containing +// item and returns the finalized File (with itemId). +export async function uploadFile( + request: APIRequestContext, + token: string, + folderId: string, + name: string, + bytes: Buffer +): Promise<{ itemId: string; itemName: string }> { + const initUrl = apiUrl( + `/file?parentType=folder&parentId=${folderId}&name=${encodeURIComponent(name)}` + + `&size=${bytes.length}&mimeType=application%2Foctet-stream` + ); + const initRes = await request.post(initUrl, { headers: { 'Girder-Token': token } }); + const upload = await readJson(initRes, `init upload ${name}`); + if (!upload?._id) throw new Error(`[e2e] init upload ${name} returned no _id`); + + const chunkUrl = apiUrl(`/file/chunk?offset=0&uploadId=${upload._id}`); + const chunkRes = await request.post(chunkUrl, { + headers: { 'Girder-Token': token, 'Content-Type': 'application/octet-stream' }, + data: bytes, + }); + const file = await readJson(chunkRes, `upload chunk ${name}`); + const itemId = file?.itemId; + if (!itemId) { + throw new Error( + `[e2e] upload of ${name} did not finalize into a File with itemId: ` + + `${JSON.stringify(file).slice(0, 300)}` + ); + } + // eslint-disable-next-line no-console + console.log(`[e2e] uploaded ${name} -> item ${itemId} (${bytes.length} bytes)`); + return { itemId, itemName: name }; +} + +// Delete a provisioned folder (teardown). +export async function deleteFolder( + request: APIRequestContext, + token: string, + folderId: string +): Promise { + const res = await request.delete(apiUrl(`/folder/${folderId}`), { + headers: { 'Girder-Token': token }, + }); + if (res.status() >= 300) { + // eslint-disable-next-line no-console + console.warn(`[e2e] delete folder ${folderId} returned HTTP ${res.status()}`); + return false; + } else { + // eslint-disable-next-line no-console + console.log(`[e2e] deleted folder ${folderId}`); + return true; + } +} diff --git a/e2e/helpers/session-zip.ts b/e2e/helpers/session-zip.ts new file mode 100644 index 0000000..fa66b9f --- /dev/null +++ b/e2e/helpers/session-zip.ts @@ -0,0 +1,67 @@ +import AdmZip from 'adm-zip'; +import { APIRequestContext } from '@playwright/test'; +import { apiUrl } from './config'; +import { readJson } from './http'; +import { ZipSummary } from './compat-state'; + +// Semantic summary of a saved session.volview.zip: counts and presence pulled +// from the manifest.json inside the archive (schema keys per VolView's +// io/state-file/schema.ts — tools.rulers.tools, segmentGroups[].path, +// parentToLayers). Deliberately NOT raw-JSON equality: the branch may migrate +// the schema, and that must stay a non-failure. + +// The newest *.volview.zip FILE inside an item (item-scoped saves append the +// session zip beside the image file; session items hold exactly one). +async function newestSessionFile( + request: APIRequestContext, + token: string, + itemId: string +): Promise<{ _id: string; name: string }> { + const res = await request.get(apiUrl(`/item/${itemId}/files?limit=100`), { + headers: { 'Girder-Token': token }, + }); + const files: Array<{ _id: string; name: string; created: string }> = await readJson( + res, + `list files of item ${itemId}` + ); + const zips = files + .filter((f) => f.name.endsWith('.volview.zip')) + .sort((a, b) => new Date(b.created).getTime() - new Date(a.created).getTime()); + if (!zips.length) throw new Error(`[compat] item ${itemId} holds no *.volview.zip file`); + return zips[0]; +} + +export async function fetchZipSummary( + request: APIRequestContext, + token: string, + itemId: string +): Promise { + const file = await newestSessionFile(request, token, itemId); + const res = await request.get(apiUrl(`/file/${file._id}/download`), { + headers: { 'Girder-Token': token }, + }); + if (res.status() >= 300) { + throw new Error(`[compat] download of ${file.name} failed: HTTP ${res.status()}`); + } + const zip = new AdmZip(await res.body()); + const manifestEntry = zip.getEntry('manifest.json'); + if (!manifestEntry) { + throw new Error(`[compat] ${file.name} has no manifest.json (not a VolView session zip?)`); + } + const manifest = JSON.parse(manifestEntry.getData().toString('utf8')); + + const segmentGroups: Array<{ path?: string }> = manifest.segmentGroups ?? []; + const groupPaths = new Set(segmentGroups.map((g) => g.path).filter(Boolean)); + const segmentGroupDataBytes = zip + .getEntries() + .filter((e) => [...groupPaths].some((p) => e.entryName === p || e.entryName.startsWith(`${p}/`))) + .reduce((max, e) => Math.max(max, e.header.size), 0); + + return { + rulerCount: manifest.tools?.rulers?.tools?.length ?? 0, + segmentGroupCount: segmentGroups.length, + segmentGroupDataBytes, + hasLayers: (manifest.parentToLayers?.length ?? 0) > 0, + version: manifest.version, + }; +} diff --git a/e2e/helpers/stack.ts b/e2e/helpers/stack.ts new file mode 100644 index 0000000..69c02d9 --- /dev/null +++ b/e2e/helpers/stack.ts @@ -0,0 +1,253 @@ +import { execFileSync } from 'child_process'; +import { createHash } from 'crypto'; +import { existsSync, readFileSync, readdirSync } from 'fs'; +import * as path from 'path'; +import { APIRequestContext } from '@playwright/test'; +import { CONFIG, apiUrl } from './config'; + +// The suite assumes an already-deployed paired stack; it does not manage docker. +// The stack must satisfy four things: +// +// 1. girder reachable at CONFIG.baseURL with CONFIG's credentials; +// 2. the backend is THIS worktree's girder_volview; +// 3. the SPA at static/built/plugins/volview/ is the paired, +// processing-enabled VolView build — a plain `girder build` re-clobbers it +// with the pinned npm package, which drops the save button and breaks the +// save/restore specs; +// 4. a deploy receipt (below) written next to the served index.html. +// +// Writing the receipt is the deploy tooling's LAST step, so its presence +// certifies the rest. + +const VERSION_URL = apiUrl('/system/version'); + +const RECEIPT_HINT = + 'Deploy the paired stack, writing deployed-heads.json next to the served ' + + 'index.html as the last step (see the contract atop this file).'; + +const BRING_UP_HINT = + `\n${RECEIPT_HINT}\n` + + `Then wait for curl -f ${VERSION_URL} to succeed.`; + +async function versionReachable(request: APIRequestContext): Promise { + try { + const res = await request.get(VERSION_URL, { timeout: 10_000 }); + return res.ok(); + } catch { + return false; + } +} + +// Single-shot health check so the tests never spin against a dead stack. +export async function healthCheck(request: APIRequestContext): Promise { + if (await versionReachable(request)) return; + throw new Error(`[e2e] girder is not reachable at ${VERSION_URL}.${BRING_UP_HINT}`); +} + +// Deploy guard. The deploy step writes a receipt recording the worktree HEADs it +// deployed. Refuse to run unless the stack serves THIS worktree's current HEAD, +// so a stale deploy fails loud instead of surfacing as a confusing mid-test +// assertion against a different checkout. +const RECEIPT_URL = `${CONFIG.baseURL}/static/built/plugins/volview/deployed-heads.json`; +const INDEX_URL = `${CONFIG.baseURL}/static/built/plugins/volview/index.html`; + +export type DeployReceipt = { + girderWorktree?: string; + girderSha?: string; + girderShort?: string; + volviewWorktree?: string; + volviewSha?: string; + volviewShort?: string; + indexMd5?: string; + backendTreeMd5?: string; +}; + +// The receipt as served (compat setup records the deployed SHAs into its state). +export async function fetchDeployReceipt(request: APIRequestContext): Promise { + const res = await request.get(RECEIPT_URL, { timeout: 10_000 }); + if (!res.ok()) throw new Error(`[e2e] no deploy receipt at ${RECEIPT_URL} (HTTP ${res.status()})`); + return JSON.parse(await res.text()); +} + +function gitHead(dir: string): string | null { + try { + return execFileSync('git', ['-C', dir, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(); + } catch { + return null; + } +} + +function md5(data: Buffer | string): string { + return createHash('md5').update(data).digest('hex'); +} + +function pythonFiles(root: string, dir = root): string[] { + const files: string[] = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + // The installed Python wheel excludes the web client's npm dependencies; + // some packages contain Python source files, which must not affect the + // backend deployment fingerprint. + if (path.relative(root, fullPath) !== path.join('web_client', 'node_modules')) { + files.push(...pythonFiles(root, fullPath)); + } + } + else if (entry.isFile() && entry.name.endsWith('.py')) files.push(fullPath); + } + return files; +} + +// Mirrors script/deploy's `find | sort | xargs md5sum | md5sum` receipt hash. +function pythonTreeMd5(root: string): string { + const digestLines = pythonFiles(root) + .map((file) => `./${path.relative(root, file).split(path.sep).join('/')}`) + .sort() + .map( + (relative) => + `${md5(readFileSync(path.join(root, relative.slice(2))))} ${relative}\n` + ) + .join(''); + return md5(digestLines); +} + +export async function verifyDeployedHeads(request: APIRequestContext): Promise { + let receipt: DeployReceipt; + try { + const res = await request.get(RECEIPT_URL, { timeout: 10_000 }); + if (!res.ok()) throw new Error(`HTTP ${res.status()}`); + receipt = JSON.parse(await res.text()); + } catch (e) { + throw new Error( + `[e2e] no deploy receipt at ${RECEIPT_URL} (${(e as Error).message}).\n` + + `Without a receipt the stack is presumed to serve stock code — not this\n` + + `worktree. ${RECEIPT_HINT}` + ); + } + + // The harness lives at /e2e/helpers, so the worktree + // root is two dirs up. Prove the stack serves THIS worktree's HEAD — unless + // E2E_EXPECT_GIRDER_SHA overrides the expectation (the compat capture phase + // runs these specs against a deliberately different deploy, e.g. main). + const override = process.env.E2E_EXPECT_GIRDER_SHA; + const worktreeRoot = path.resolve(__dirname, '..', '..'); + const expected = override || gitHead(worktreeRoot); + if (override) { + // eslint-disable-next-line no-console + console.log( + `[e2e] E2E_EXPECT_GIRDER_SHA set: expecting deployed girder ${override.slice(0, 9)}` + ); + } + // Both halves of the comparison must exist, or the guard is not a guard: a + // missing sha on either side is when a wrong deploy is most likely, not least. + if (!receipt.girderSha) { + throw new Error( + `[e2e] the deploy receipt at ${RECEIPT_URL} has no girderSha, so it cannot ` + + `certify what the stack serves. ${RECEIPT_HINT}` + ); + } + if (!expected) { + throw new Error( + `[e2e] cannot determine the expected girder_volview sha: ${worktreeRoot} is not a\n` + + `git checkout and E2E_EXPECT_GIRDER_SHA is unset. Set E2E_EXPECT_GIRDER_SHA to the\n` + + `sha the stack should be serving (the compat harness does this for its baseline,\n` + + `which is a git-archive export with no .git of its own).` + ); + } + if (expected !== receipt.girderSha) { + throw new Error( + `[e2e] deploy is stale: the stack serves girder_volview ${receipt.girderShort} ` + + `but the expected sha is ${expected.slice(0, 9)}` + + `${override ? ' (from E2E_EXPECT_GIRDER_SHA)' : ' (this worktree HEAD)'}.\n` + + `Redeploy and refresh the receipt. ${RECEIPT_HINT}` + ); + } + + const volviewOverride = process.env.E2E_EXPECT_VOLVIEW_SHA; + const receiptVolviewHead = receipt.volviewWorktree + ? gitHead(receipt.volviewWorktree) + : null; + const expectedVolview = volviewOverride || receiptVolviewHead; + if (volviewOverride) { + // eslint-disable-next-line no-console + console.log( + `[e2e] E2E_EXPECT_VOLVIEW_SHA set: expecting deployed VolView ${volviewOverride.slice(0, 9)}` + ); + } + if (!receipt.volviewSha) { + throw new Error( + `[e2e] the deploy receipt at ${RECEIPT_URL} has no volviewSha, so it cannot ` + + `certify which client the stack serves. ${RECEIPT_HINT}` + ); + } + if (!expectedVolview) { + throw new Error( + `[e2e] cannot determine the expected VolView sha: the receipt's VolView worktree ` + + `is unavailable and E2E_EXPECT_VOLVIEW_SHA is unset. Set E2E_EXPECT_VOLVIEW_SHA ` + + `to the client sha the stack should serve.` + ); + } + if (expectedVolview !== receipt.volviewSha) { + throw new Error( + `[e2e] deploy is stale: the stack serves VolView ${receipt.volviewShort}, but the ` + + `expected sha is ${expectedVolview.slice(0, 9)}` + + `${volviewOverride ? ' (from E2E_EXPECT_VOLVIEW_SHA)' : ' (receipt worktree HEAD)'}.\n` + + `Redeploy and refresh the receipt. ${RECEIPT_HINT}` + ); + } + + if (!receipt.indexMd5) { + throw new Error(`[e2e] deploy receipt has no indexMd5. ${RECEIPT_HINT}`); + } + const servedIndex = await request.get(INDEX_URL, { timeout: 10_000 }); + if (!servedIndex.ok()) { + throw new Error( + `[e2e] cannot read deployed VolView index at ${INDEX_URL} (HTTP ${servedIndex.status()})` + ); + } + const servedIndexMd5 = md5(await servedIndex.body()); + if (servedIndexMd5 !== receipt.indexMd5) { + throw new Error( + `[e2e] deployed VolView index hash is ${servedIndexMd5}, but the receipt records ` + + `${receipt.indexMd5}. Redeploy and refresh the receipt.` + ); + } + + if (receipt.volviewWorktree && existsSync(receipt.volviewWorktree)) { + const builtIndex = path.join(receipt.volviewWorktree, 'dist', 'index.html'); + if (!existsSync(builtIndex)) { + throw new Error( + `[e2e] receipt VolView worktree has no built index: ${builtIndex}` + ); + } + const builtIndexMd5 = md5(readFileSync(builtIndex)); + if (builtIndexMd5 !== receipt.indexMd5) { + throw new Error( + `[e2e] VolView worktree build hash is ${builtIndexMd5}, but the deployed receipt ` + + `records ${receipt.indexMd5}. Redeploy before running the browser suite.` + ); + } + } + + if (!receipt.backendTreeMd5) { + throw new Error(`[e2e] deploy receipt has no backendTreeMd5. ${RECEIPT_HINT}`); + } + // The receipt's backendTreeMd5 covers the installed PACKAGE directory, so hash + // the worktree's package directory too -- hashing the worktree root sweeps in + // setup.py, tests/ and e2e/seed/, which can never match. + const backendPkg = path.join(receipt.girderWorktree ?? '', 'girder_volview'); + if (receipt.girderWorktree && existsSync(backendPkg)) { + const currentBackendMd5 = pythonTreeMd5(backendPkg); + if (currentBackendMd5 !== receipt.backendTreeMd5) { + throw new Error( + `[e2e] girder_volview worktree hash is ${currentBackendMd5}, but the deployed ` + + `receipt records ${receipt.backendTreeMd5}. Redeploy before running the browser suite.` + ); + } + } + + // eslint-disable-next-line no-console + console.log( + `[e2e] deploy receipt OK — girder ${receipt.girderShort}, VolView ${receipt.volviewShort}.` + ); +} diff --git a/e2e/helpers/volview.ts b/e2e/helpers/volview.ts new file mode 100644 index 0000000..22ef3b2 --- /dev/null +++ b/e2e/helpers/volview.ts @@ -0,0 +1,143 @@ +import { Page, TestInfo, expect } from '@playwright/test'; + +// VolView is ready once a vtk view canvas has real (non-zero) dimensions — the +// same signal the VolView wdio suite uses (waitForViews). +export async function waitForVolViewReady(page: Page, timeout = 90_000) { + await page.locator('[data-testid~="vtk-view"] canvas').first().waitFor({ state: 'attached', timeout }); + await page.waitForFunction( + () => { + const canvases = document.querySelectorAll('[data-testid~="vtk-view"] canvas'); + return Array.from(canvases).some( + (c) => (c as HTMLCanvasElement).width > 10 && (c as HTMLCanvasElement).height > 10 + ); + }, + undefined, + { timeout } + ); + // Let the first frame settle for a faithful screenshot. + await page.waitForTimeout(1500); +} + +// The current value of the tab's `urls=` launch param (decoded). This is where +// resume-vs-fresh lives: F5 re-fetches exactly this. +export function urlsParam(page: Page): string { + const u = new URL(page.url()); + return u.searchParams.get('urls') || ''; +} + +// Screenshot into the test's output dir with a stable, human-scannable name. +export async function shot(page: Page, info: TestInfo, name: string) { + const file = info.outputPath(`${name}.png`); + await page.screenshot({ path: file, fullPage: false }); + await info.attach(name, { path: file, contentType: 'image/png' }); +} + +// Trigger the girder-launched REMOTE save (ControlsStrip save button -> +// remote-save-state.saveState() -> POST save= -> resumeUrl repoint). Returns the +// resumeUrl the backend handed back. No dialog is shown when save= is set. +export async function remoteSave(page: Page): Promise { + const saveButton = page.locator('button:has(i.mdi-content-save-all)').first(); + await expect(saveButton, 'save button (mdi-content-save-all) not found').toBeVisible(); + + const savePost = page.waitForResponse( + (r) => + r.request().method() === 'POST' && + /\/(item|folder)\/[^/]+\/volview(\?|$)/.test(r.url()), + { timeout: 60_000 } + ); + await saveButton.click(); + // Main-era clients interpose a "Saving Session State" filename dialog even + // for remote saves; confirm it. The branch client saves directly, so the + // button simply never appears. + const confirmSave = page.locator('[data-testid="save-session-confirm-button"]').first(); + if (await confirmSave.isVisible({ timeout: 3_000 }).catch(() => false)) { + await confirmSave.click(); + } + const res = await savePost; + expect(res.status(), `save POST failed: ${res.status()} ${res.url()}`).toBeLessThan(300); + + let resumeUrl = ''; + try { + const body = await res.json(); + resumeUrl = body?.resumeUrl || ''; + } catch { + /* fail-safe: some responses may not be JSON */ + } + // The client repoints urls= via history.replaceState after a resumeUrl. + if (resumeUrl) { + await expect + .poll(() => urlsParam(page), { timeout: 15_000, message: 'urls= did not repoint to resumeUrl' }) + .toBe(resumeUrl); + } + // Best-effort: the "Save Successful" toast. + await page + .getByText('Save Successful', { exact: false }) + .waitFor({ state: 'visible', timeout: 10_000 }) + .catch(() => undefined); + return resumeUrl; +} + +// Click a module tab by name (Jobs / Annotations / Rendering / Data). The Jobs +// tab appears only when the launch config= registered a processing provider; +// its job list is folder+user scoped. +export async function openModuleTab(page: Page, name: string): Promise { + await page.locator(`button[data-testid="module-tab-${name}"]`).click(); +} + +// Open the Jobs tab and click the first succeeded job's "Load" — the come-back +// path: this fetches the results AND applies them through the same +// intent-honoring pipeline the live flow uses (labelmap → segment group on the +// reconstructed parent image, plain image → new dataset). The button is +// consumed once the scene application finishes. +export async function loadJobResults(page: Page): Promise { + await openModuleTab(page, 'Jobs'); + const panel = page.locator('.jobs-module'); + const load = panel.getByRole('button', { name: 'Load', exact: true }).first(); + await expect(load, 'no "Load" button — is the succeeded job listed in the Jobs tab?').toBeVisible(); + await load.click(); + await expect(load, 'the job result did not finish applying').toHaveCount(0); +} + +// Select a registered task in the Jobs tab's TaskPicker (a v-select labelled +// "Task"), matching by title prefix (e.g. "Otsu"). Vuetify's floating label is +// not programmatically associated with the input, so match the v-select itself. +export async function selectTask(page: Page, titlePrefix: string): Promise { + await openModuleTab(page, 'Jobs'); + const picker = page.locator('.jobs-module .v-select', { hasText: 'Task' }).first(); + await expect(picker, 'no Task picker — is a processing provider registered?').toBeVisible(); + await picker.click(); + const option = page + .locator('.v-overlay-container .v-list-item, .v-menu .v-list-item') + .filter({ hasText: titlePrefix }) + .first(); + await expect(option, `no "${titlePrefix}*" task option in the picker`).toBeVisible(); + await option.click(); +} + +// Wait for the auto-bound image input to render (FileWidget's "Active dataset" +// caption under the bound image name) before submit — a volume with no server +// provenance blocks submit, so this proves the binding step ran. +export async function waitForInputBound(page: Page, timeout = 30_000): Promise { + await expect( + page.locator('.jobs-module').getByText('Active dataset').first(), + 'the image input never bound to the active dataset' + ).toBeVisible({ timeout }); +} + +export async function submitTaskFromForm(page: Page): Promise { + const submit = page + .locator('.jobs-module') + .getByRole('button', { name: 'Submit', exact: true }); + await expect(submit, 'Submit button not enabled (unbound input / form invalid?)').toBeEnabled(); + await submit.click(); +} + +// Wait for the live completion toast ("Job complete: ") the store raises +// once a job submitted THIS session reaches success — the live path, not a +// re-discovered history row. +export async function waitForJobComplete(page: Page, timeout = 180_000): Promise { + await expect( + page.locator('.Vue-Toastification__toast', { hasText: 'Job complete' }).first(), + 'no "Job complete" toast — did the live job finish?' + ).toBeVisible({ timeout }); +} diff --git a/e2e/package-lock.json b/e2e/package-lock.json new file mode 100644 index 0000000..e593175 --- /dev/null +++ b/e2e/package-lock.json @@ -0,0 +1,133 @@ +{ + "name": "girder-volview-e2e", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "girder-volview-e2e", + "version": "0.1.0", + "devDependencies": { + "@playwright/test": "^1.48.0", + "@types/adm-zip": "^0.5.8", + "@types/node": "^20.0.0", + "adm-zip": "^0.6.0", + "typescript": "^5.6.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", + "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/adm-zip": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.5.8.tgz", + "integrity": "sha512-RVVH7QvZYbN+ihqZ4kX/dMiowf6o+Jk1fNwiSdx0NahBJLU787zkULhGhJM8mf/obmLGmgdMM0bXsQTmyfbR7Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/node": { + "version": "20.19.43", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz", + "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/adm-zip": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.0.tgz", + "integrity": "sha512-XleryMhbuksdKtofnWZ9Sk+4CUTbms4Mb/EU32SZwToAyZ5RgVos/ki8n+yr0LWHOGKuakbXTuuYNHLQjhddgg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/e2e/package.json b/e2e/package.json new file mode 100644 index 0000000..e766253 --- /dev/null +++ b/e2e/package.json @@ -0,0 +1,26 @@ +{ + "name": "girder-volview-e2e", + "version": "0.1.0", + "private": true, + "description": "Browser e2e for girder_volview + VolView.", + "scripts": { + "test": "bash scripts/compat.sh", + "typecheck": "tsc --noEmit", + "list": "playwright test --list", + "install-browser": "playwright install chromium", + "report": "playwright show-report", + "compat": "bash scripts/compat.sh", + "compat:capture": "bash scripts/compat.sh --phase capture", + "compat:verify": "bash scripts/compat.sh --phase verify", + "compat:verify-fast": "bash scripts/compat.sh --phase verify --skip-deploy --keep", + "current": "bash scripts/compat.sh --phase current --skip-deploy", + "compat:clean": "bash ../script/clean-compat" + }, + "devDependencies": { + "@playwright/test": "^1.48.0", + "@types/adm-zip": "^0.5.8", + "@types/node": "^20.0.0", + "adm-zip": "^0.6.0", + "typescript": "^5.6.0" + } +} diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts new file mode 100644 index 0000000..af5f7d1 --- /dev/null +++ b/e2e/playwright.config.ts @@ -0,0 +1,43 @@ +import { defineConfig, devices } from '@playwright/test'; +import { CONFIG } from './helpers/config'; + +// One browser harness, invoked in phases by scripts/compat.sh: +// baseline capture -> current verification -> current-only behavior. +export default defineConfig({ + testDir: './tests', + globalSetup: require.resolve('./compat.setup'), + globalTeardown: require.resolve('./compat.teardown'), + timeout: 240_000, + expect: { timeout: 45_000 }, + fullyParallel: false, + workers: 1, + retries: 0, + reporter: [['list'], ['html', { open: 'never', outputFolder: 'playwright-report' }]], + outputDir: 'test-results', + use: { + baseURL: CONFIG.baseURL, + headless: true, + screenshot: 'on', + trace: 'on', + video: 'retain-on-failure', + ignoreHTTPSErrors: true, + viewport: { width: 1600, height: 1000 }, + }, + projects: [ + { + name: 'capture', + testMatch: /.*capture\.spec\.ts/, + use: { ...devices['Desktop Chrome'] }, + }, + { + name: 'verify', + testMatch: /.*verify\.spec\.ts/, + use: { ...devices['Desktop Chrome'] }, + }, + { + name: 'current', + testMatch: /tests\/(?!compat\/).*\.spec\.ts$/, + use: { ...devices['Desktop Chrome'] }, + }, + ], +}); diff --git a/e2e/scripts/compat.sh b/e2e/scripts/compat.sh new file mode 100755 index 0000000..8bdbeb6 --- /dev/null +++ b/e2e/scripts/compat.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Backwards-compat orchestration: +# +# 1. materialize the BASELINE girder_volview from git history (no second +# checkout required) and deploy it with the baseline VolView +# 2. playwright `capture` project — real-UI gestures, content, saves +# (expected backend + client shas carry the pins past the deploy guard) +# 3. redeploy THIS worktree + its paired VolView +# 4. playwright `verify` project — sessions must restore + re-save +# 5. playwright `current` project — fresh current sessions, restart/history, +# grouped DICOM launches, and job submission, all on isolated folders +# +# The stack itself (a running docker compose project, dsa-plus by default) must +# already exist; script/deploy only swaps the code it serves. Mongo survives the +# redeploy, so the girder folders/sessions captured in step 2 are still there +# for step 4. +# +# The baseline is a `git archive` export under the gitignored e2e/.compat/, +# pinned by e2e/compat-baseline.json. Neither the old sources nor the session +# zips they produce are ever committed — both are reproducible from a sha. +# +# Usage: compat.sh [--phase all|capture|verify|current] [--skip-deploy] [--link] [--keep] +# +# --phase which half to run (default all) +# --skip-deploy don't deploy (the stack already serves the right code) +# --link pass --link to script/deploy (fast client copy; default pack) +# --keep keep the run folder + state after verify (iteration) +# +# Env overrides: COMPAT_BASELINE_REF (baseline ref instead of the pin), +# COMPAT_NO_FETCH, COMPAT_OLD_CHECKOUT/COMPAT_OLD_SHA, COMPAT_BRANCH_VOLVIEW, +# COMPAT_BASELINE_VOLVIEW, COMPAT_BRANCH_VOLVIEW_SHA, +# COMPAT_BASELINE_VOLVIEW_SHA, COMPAT_DEPLOY. + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO=$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel) +E2E="$REPO/e2e" +MANIFEST="$E2E/compat-baseline.json" + +BASELINE_VOLVIEW=${COMPAT_BASELINE_VOLVIEW:-main} +BRANCH_VOLVIEW=${COMPAT_BRANCH_VOLVIEW:-just-jobs} +DEPLOY=${COMPAT_DEPLOY:-$REPO/script/deploy} + +PHASE=all +SKIP_DEPLOY=0 +LINK_FLAG="" +KEEP=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --phase) PHASE=$2; shift 2 ;; + --skip-deploy) SKIP_DEPLOY=1; shift ;; + --link) LINK_FLAG=--link; shift ;; + --keep) KEEP=1; shift ;; + -h|--help) sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown flag: $1" >&2; exit 2 ;; + esac +done +case "$PHASE" in all|capture|verify|current) ;; *) echo "--phase must be all|capture|verify|current" >&2; exit 2 ;; esac + +die() { echo "compat: $*" >&2; exit 1; } + +manifest_sha() { + python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))[sys.argv[2]]["sha"])' \ + "$MANIFEST" "$1" || die "could not read $1.sha from $MANIFEST" +} + +resolve_volview() { + local arg=$1 + if [[ $arg = /* ]]; then + printf '%s\n' "$arg" + elif [[ -d $arg ]]; then + realpath "$arg" + else + : "${VOLVIEW_ROOT:?set VOLVIEW_ROOT in .env or pass an explicit VolView checkout path}" + printf '%s/%s\n' "$VOLVIEW_ROOT" "$arg" + fi +} + +require_volview_sha() { + local checkout=$1 expected=$2 label=$3 actual + [[ -f $checkout/package.json ]] || die "$label VolView checkout not found: $checkout" + actual=$(git -C "$checkout" rev-parse HEAD 2>/dev/null) || \ + die "$label VolView path is not a git checkout: $checkout" + [[ $actual = "$expected" ]] || \ + die "$label VolView is at $actual, but expected $expected ($checkout)" +} + +[[ -x $DEPLOY ]] || die "deploy script not found/executable: $DEPLOY (set COMPAT_DEPLOY)" +command -v uv >/dev/null || die "uv is required (seed.py seed-small runs via 'uv run')" +[[ -f $MANIFEST ]] || die "missing $MANIFEST" + +BASELINE_VOLVIEW_SHA=${COMPAT_BASELINE_VOLVIEW_SHA:-$(manifest_sha volview)} +[[ $BASELINE_VOLVIEW_SHA =~ ^[0-9a-f]{40}$ ]] || die "invalid baseline VolView sha: $BASELINE_VOLVIEW_SHA" + +if [[ -f $REPO/.env ]]; then + set -a + # shellcheck disable=SC1091 + . "$REPO/.env" + set +a +fi + +if [[ $SKIP_DEPLOY -eq 0 && ($PHASE == all || $PHASE == capture) ]]; then + BASELINE_VOLVIEW=$(resolve_volview "$BASELINE_VOLVIEW") + require_volview_sha "$BASELINE_VOLVIEW" "$BASELINE_VOLVIEW_SHA" baseline +fi +if [[ $PHASE == all || $PHASE == verify || $PHASE == current ]]; then + BRANCH_VOLVIEW=$(resolve_volview "$BRANCH_VOLVIEW") + BRANCH_VOLVIEW_HEAD=$(git -C "$BRANCH_VOLVIEW" rev-parse HEAD 2>/dev/null) || \ + die "branch VolView path is not a git checkout: $BRANCH_VOLVIEW" + BRANCH_VOLVIEW_SHA=${COMPAT_BRANCH_VOLVIEW_SHA:-$BRANCH_VOLVIEW_HEAD} + [[ $BRANCH_VOLVIEW_SHA =~ ^[0-9a-f]{40}$ ]] || die "invalid branch VolView sha: $BRANCH_VOLVIEW_SHA" + require_volview_sha "$BRANCH_VOLVIEW" "$BRANCH_VOLVIEW_SHA" branch +fi + +# Unconditional, including under --skip-deploy: the capture phase exports +# E2E_EXPECT_GIRDER_SHA either way, and this is a cache hit that needs no docker. +BASELINE_DIR_SHA=$("$E2E/scripts/materialize-baseline.sh") +MAIN_SHA=$BASELINE_DIR_SHA +BASELINE_DIR="$E2E/.compat/checkout-${MAIN_SHA:0:9}" +[[ -n ${COMPAT_OLD_CHECKOUT:-} ]] && BASELINE_DIR=$COMPAT_OLD_CHECKOUT +CUSTOM_BASELINE=0 +[[ -n ${COMPAT_OLD_CHECKOUT:-} ]] && CUSTOM_BASELINE=1 + +BRANCH_SHA=$(git -C "$REPO" rev-parse HEAD) +if [[ $MAIN_SHA == "$BRANCH_SHA" ]]; then + echo "compat: WARNING — the baseline and this worktree are the same commit; the run is vacuous" >&2 +fi + +echo "compat: baseline ${MAIN_SHA:0:9} at $BASELINE_DIR (VolView: ${BASELINE_VOLVIEW_SHA:0:9} at $BASELINE_VOLVIEW)" +if [[ $PHASE == all || $PHASE == verify || $PHASE == current ]]; then + echo "compat: branch ${BRANCH_SHA:0:9} at $REPO (VolView: ${BRANCH_VOLVIEW_SHA:0:9} at $BRANCH_VOLVIEW)" +fi + +run_capture() { + echo "compat: ensuring the small-tier DICOM cache (fetch --small is idempotent)..." + uv run "$E2E/seed/seed.py" fetch --small + + if [[ $SKIP_DEPLOY -eq 0 ]]; then + echo "compat: deploying the baseline..." + if [[ $CUSTOM_BASELINE -eq 1 ]]; then + # A custom baseline is a real git checkout whose HEAD was verified + # by materialize-baseline.sh, so deploy derives its receipt normally. + "$DEPLOY" $LINK_FLAG -- "$BASELINE_DIR" "$BASELINE_VOLVIEW" + else + # The normal export is a plain tree with no .git to ask. + "$DEPLOY" $LINK_FLAG --girder-sha "$MAIN_SHA" -- "$BASELINE_DIR" "$BASELINE_VOLVIEW" + fi + fi + echo "compat: running capture specs against the baseline (${MAIN_SHA:0:9})..." + ( + cd "$E2E" + COMPAT_PHASE=capture E2E_EXPECT_GIRDER_SHA=$MAIN_SHA \ + E2E_EXPECT_VOLVIEW_SHA=$BASELINE_VOLVIEW_SHA \ + npx playwright test --config playwright.config.ts --project capture + ) +} + +run_verify() { + if [[ $SKIP_DEPLOY -eq 0 ]]; then + echo "compat: deploying THIS worktree..." + "$DEPLOY" $LINK_FLAG "$REPO" "$BRANCH_VOLVIEW" + fi + echo "compat: running verify specs against this worktree (${BRANCH_SHA:0:9})..." + ( + cd "$E2E" + if [[ $KEEP -eq 1 ]]; then export COMPAT_KEEP=1; fi + if [[ $PHASE == verify ]]; then export COMPAT_CLEANUP=1; fi + COMPAT_PHASE=verify E2E_EXPECT_GIRDER_SHA=$BRANCH_SHA \ + E2E_EXPECT_VOLVIEW_SHA=$BRANCH_VOLVIEW_SHA \ + npx playwright test --config playwright.config.ts --project verify + ) +} + +run_current() { + echo "compat: running current-version lifecycle and job scenarios (${BRANCH_SHA:0:9})..." + ( + cd "$E2E" + if [[ $KEEP -eq 1 ]]; then export COMPAT_KEEP=1; fi + COMPAT_PHASE=current COMPAT_CLEANUP=1 E2E_EXPECT_GIRDER_SHA=$BRANCH_SHA \ + E2E_EXPECT_VOLVIEW_SHA=$BRANCH_VOLVIEW_SHA \ + npx playwright test --config playwright.config.ts --project current + ) +} + +if [[ $PHASE == all || $PHASE == capture ]]; then run_capture; fi +if [[ $PHASE == all || $PHASE == verify ]]; then run_verify; fi +if [[ $PHASE == all || $PHASE == current ]]; then run_current; fi + +echo "compat: done — source sessions verified and current lifecycles exercised on ${BRANCH_SHA:0:9}." +echo "compat: report: cd e2e && npm run report" diff --git a/e2e/scripts/materialize-baseline.sh b/e2e/scripts/materialize-baseline.sh new file mode 100755 index 0000000..e9693c4 --- /dev/null +++ b/e2e/scripts/materialize-baseline.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Recreate the baseline (old) girder_volview tree from git history. +# +# The compat suite needs to deploy an OLDER version of this plugin so it can +# save sessions the way that version did. The old tree is derived from the repo's +# own history so the suite does not depend on a second checkout existing on the +# developer's machine. +# +# `git archive` rather than `git worktree add`: the export is ~36 files, it +# takes milliseconds, and — the deciding reason — it leaves no entry in the +# shared .git/worktrees registry. A crashed run therefore cannot strand a +# registration that a later `worktree add` trips over, and the tree can be +# deleted with plain rm once ownership is sane. The cost is that the export has +# no .git, so its sha must be passed to the deploy explicitly; script/deploy +# still proves the mounted backend matches the tree it was handed. +# +# Nothing this produces is ever committed: e2e/.compat/ is gitignored, because a +# checkout reproducible from a sha is not source. +# +# Usage: materialize-baseline.sh +# stdout: the resolved 40-char sha, and nothing else (callers capture it) +# stderr: progress +# +# Env: +# COMPAT_BASELINE_REF resolve this ref instead of the pinned sha +# COMPAT_NO_FETCH=1 skip the `git fetch` refresh (offline) +# COMPAT_OLD_CHECKOUT use this git checkout as-is; requires COMPAT_OLD_SHA +# COMPAT_OLD_SHA required HEAD of COMPAT_OLD_CHECKOUT + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO=$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel) +MANIFEST="$REPO/e2e/compat-baseline.json" + +die() { echo "materialize-baseline: $*" >&2; exit 1; } + +# Escape hatch for iterating on a baseline that is not a committed ref (the +# PostgreSQL cross-version suite exposes the same seam as `oldinstall`). +if [ -n "${COMPAT_OLD_CHECKOUT:-}" ]; then + [ -n "${COMPAT_OLD_SHA:-}" ] || die "COMPAT_OLD_CHECKOUT requires COMPAT_OLD_SHA" + [ -f "$COMPAT_OLD_CHECKOUT/setup.py" ] || die "COMPAT_OLD_CHECKOUT is not a girder_volview tree: $COMPAT_OLD_CHECKOUT" + ACTUAL_SHA=$(git -C "$COMPAT_OLD_CHECKOUT" rev-parse HEAD 2>/dev/null) || \ + die "COMPAT_OLD_CHECKOUT must be a git checkout: $COMPAT_OLD_CHECKOUT" + EXPECTED_SHA=$(git -C "$COMPAT_OLD_CHECKOUT" rev-parse --verify "$COMPAT_OLD_SHA^{commit}" 2>/dev/null) || \ + die "COMPAT_OLD_SHA is not a commit in $COMPAT_OLD_CHECKOUT: $COMPAT_OLD_SHA" + [ "$ACTUAL_SHA" = "$EXPECTED_SHA" ] || \ + die "COMPAT_OLD_CHECKOUT is at $ACTUAL_SHA, expected $EXPECTED_SHA" + echo "materialize-baseline: using COMPAT_OLD_CHECKOUT=$COMPAT_OLD_CHECKOUT (${ACTUAL_SHA:0:9})" >&2 + echo "$ACTUAL_SHA" + exit 0 +fi + +[ -f "$MANIFEST" ] || die "missing $MANIFEST" + +REF=${COMPAT_BASELINE_REF:-} +if [ "${COMPAT_NO_FETCH:-0}" != 1 ]; then + git -C "$REPO" fetch --quiet origin main 2>/dev/null || \ + echo "materialize-baseline: fetch failed; falling back to local objects" >&2 +fi + +if [ -n "$REF" ]; then + SHA=$(git -C "$REPO" rev-parse --verify "$REF^{commit}" 2>/dev/null) || \ + die "cannot resolve COMPAT_BASELINE_REF=$REF" + echo "materialize-baseline: COMPAT_BASELINE_REF=$REF -> ${SHA:0:9} (overriding the pin)" >&2 +else + SHA=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["girder"]["sha"])' "$MANIFEST") || \ + die "could not read girder.sha from $MANIFEST" + git -C "$REPO" cat-file -e "$SHA^{commit}" 2>/dev/null || \ + die "pinned baseline $SHA is not in this clone. Fetch it: git fetch origin main" +fi + +DEST="$REPO/e2e/.compat/checkout-${SHA:0:9}" + +# Keyed on the sha, so bumping the pin extracts into a virgin directory instead +# of needing the old one removed first — which matters because the old one may +# still be root-owned from a container mount. +if [ -f "$DEST/setup.py" ]; then + echo "materialize-baseline: reusing $DEST" >&2 +else + rm -rf "$DEST" + mkdir -p "$DEST" + git -C "$REPO" archive "$SHA" | tar -x -C "$DEST" + echo "materialize-baseline: exported ${SHA:0:9} -> $DEST" >&2 +fi + +# A truncated export would deploy a half-plugin and fail much later as a +# confusing test error. `git archive` honours .gitattributes export-ignore, so +# this also catches someone adding one. +[ -f "$DEST/setup.py" ] && [ -d "$DEST/girder_volview" ] || \ + die "export at $DEST is missing setup.py or girder_volview/ — delete it and retry" + +printf '{"resolvedSha":"%s","requestedRef":"%s"}\n' "$SHA" "${REF:-pinned}" > "$DEST/.compat-source.json" + +echo "$SHA" diff --git a/e2e/seed/ATTRIBUTION.md b/e2e/seed/ATTRIBUTION.md new file mode 100644 index 0000000..597da3c --- /dev/null +++ b/e2e/seed/ATTRIBUTION.md @@ -0,0 +1,46 @@ +# Attribution + +The DICOM data this devkit downloads comes from the NCI Imaging Data Commons +(IDC) and is redistributed under CC BY 3.0 / CC BY 4.0. No imaging data is +stored in this repository -- `seed.py fetch` pulls it from IDC's public bucket. + +## Datasets used + +- Kinahan, P., Muzi, M., Bialecki, B., Herman, B., & Coombs, L. (2019). Data from the ACRIN 6668 Trial NSCLC-FDG-PET (Version 2) [Dataset]. The Cancer Imaging Archive. https://doi.org/10.7937/TCIA.2019.30ILQFCL +- Eisenbrey, J., Lyshchik, A., & Wessner, C. (2021). Ultrasound data of a variety of liver masses (Version 1) [Dataset]. The Cancer Imaging Archive. https://doi.org/10.7937/TCIA.2021.V4Z7-TC39 +- Fedorov, A., Longabaugh, W. J. R., Pot, D., Clunie, D. A., Pieper, S. D., Gibbs, D. L., Bridge, C., Herrmann, M. D., Homeyer, A., Lewis, R., Aerts, H. J. W. L., Krishnaswamy, D., Thiriveedhi, V. K., Ciausu, C., Schacherer, D. P., Bontempi, D., Pihl, T., Wagner, U., Farahani, K., et al. (2023). National Cancer Institute Imaging Data Commons: Toward Transparency, Reproducibility, and Scalability in Imaging Artificial Intelligence. RadioGraphics, 43(12). https://doi.org/10.1148/rg.230180 +- Litjens, G., Debats, O., Barentsz, J., Karssemeijer, N., & Huisman, H. + (2017). *SPIE-AAPM PROSTATEx Challenge Data* (Version 2) [Dataset]. The + Cancer Imaging Archive. https://doi.org/10.7937/K9TCIA.2017.MURS5CL + +IDC selection DOIs: 10.7937/tcia.2019.30ilqfcl, 10.7937/tcia.2021.v4z7-tc39 + +## VolView developer examples + +The prostate subset and fetal ultrasound volume are pinned by SHA-512 and +downloaded from Kitware's public VolView example-data folder: + +- `MRI-PROSTATEx-0004.zip` +- `prostate-total.seg.nii.gz` (converted locally to `.seg.nrrd`) +- `3DUS-Fetus.mha` + +The fetal segmentation is generated locally with Otsu thresholding; it is not +a clinical annotation. + +## IDC + +Fedorov, A., Longabaugh, W. J. R., Pot, D., et al. *National Cancer Institute +Imaging Data Commons: Toward Transparency, Reproducibility, and Scalability in +Imaging Artificial Intelligence.* RadioGraphics (2023). +https://doi.org/10.1148/rg.230180 + +## Terms that travel with this data + +Per the TCIA Data Usage Policy +(https://www.cancerimagingarchive.net/data-usage-policies-and-restrictions/): + +- Attribute each individual dataset used, and link to that policy. +- Pass this same obligation on to downstream users. +- Do not attempt to identify or contact the individuals these images came from. + +Regenerate this file with `uv run seed.py fetch`. diff --git a/e2e/seed/README.md b/e2e/seed/README.md new file mode 100644 index 0000000..0888df8 --- /dev/null +++ b/e2e/seed/README.md @@ -0,0 +1,61 @@ +# VolView manual-test seed + +Seeds three Girder collections with real public imaging data through an S3 +import: + +```text +Trial +├── patients///{CT,PET}/ +└── ultrasound/clip-{01,02,03}.dcm + +Trial (Large Image Filter) +├── patients///{CT,PET}/ +└── ultrasound/clip-{01,02,03}.dcm + +Developer +├── prostate/{dicom/,5.seg.total-segmentator.nrrd} +├── fetus/{fetus.mha,fetus.seg.nrrd} +└── ultrasound/clip-{01,02,03}.dcm +``` + +The trial collections mirror one another: three patients, two studies per +patient, and CT plus PET in every study. Only the second collection receives +`.large_image_config.yaml`, in both `patients/` and `ultrasound/`. The Trial and +Developer ultrasound folders do not use large-image filtering. + +For **Open Checked in VolView**, select `prostate/dicom` with its segmentation, +or select both files in `fetus`. The fetal segmentation is generated test data, +not a clinical annotation. + +## Run + +Start MinIO in the existing `dsa-plus` Compose project: + +```bash +docker compose -p dsa-plus -f docker-compose.minio.yml up -d +``` + +Then prepare and seed: + +```bash +uv run seed.py fetch +uv run seed.py stage +uv run seed.py seed +uv run seed.py verify +``` + +To clean and recreate all three managed collections while keeping the download +cache and staged MinIO objects: + +```bash +uv run seed.py reseed +uv run seed.py verify +``` + +`reset` deletes the collections. Use `reset --bucket` to also empty MinIO. +`stage --max-slices N` controls the number of CT/PET instances per series; the +default is 40. + +No imaging data is committed. Downloads live under ignored `data/` storage and +are pinned by SeriesInstanceUID or SHA-512. See +[ATTRIBUTION.md](ATTRIBUTION.md) for sources and terms. diff --git a/e2e/seed/configs/developer/.volview_config.yaml b/e2e/seed/configs/developer/.volview_config.yaml new file mode 100644 index 0000000..5f7b75e --- /dev/null +++ b/e2e/seed/configs/developer/.volview_config.yaml @@ -0,0 +1,31 @@ +# VolView app config for the ad hoc developer examples. The `.seg.nrrd` +# naming convention associates each labelmap with the image checked beside it. + +disabledViewTypes: + - Oblique + +layouts: + Axial Coronal Sagittal: + direction: row + items: + - axial + - direction: column + items: + - coronal + - sagittal + Quad View: + direction: column + items: + - direction: row + items: + - axial + - coronal + - direction: row + items: + - sagittal + - volume + +io: + segmentGroupExtension: seg + segmentGroupSaveFormat: nrrd + layerExtension: layer diff --git a/e2e/seed/configs/trial/.large_image_config.yaml b/e2e/seed/configs/trial/.large_image_config.yaml new file mode 100644 index 0000000..a2e9924 --- /dev/null +++ b/e2e/seed/configs/trial/.large_image_config.yaml @@ -0,0 +1,153 @@ +# Clinical-trial hierarchy view: Patient rows -> Study rows -> open in VolView. +# +# The S3 import creates one Girder item per .dcm key, so without this file the +# trial folder renders as a flat wall of thousands of slices. `group` collapses +# those into one row per patient; `layout.flatten: only` pulls items up from the +# nested patient/study/CT|PET folders and hides the folders themselves. +# +# Grouping keys resolve against item metadata as `meta.dicom.*`. girder_volview +# populates the instance tags on both uploads and S3 imports; `seed.py seed` +# adds the derived `ModalitiesInStudy` value that is absent from each instance. + +defaultItemList: patientList + +itemList: &plainColumns + columns: + - type: image + value: thumbnail + title: Thumbnail + - type: record + value: name + title: Name + - type: record + value: controls + title: Controls + - type: record + value: size + title: Size + +itemListDialog: *plainColumns + +namedItemLists: + patientList: + layout: + flatten: only + group: + keys: dicom.PatientID + counts: + _id: _count.slicescount + dicom.SeriesInstanceUID: _count.seriescount + dicom.StudyInstanceUID: _count.studiescount + navigate: + type: itemList + name: studyList + defaultSort: + - type: metadata + value: dicom.PatientID + dir: up + columns: + - type: image + value: thumbnail + title: Thumbnail + - type: metadata + value: dicom.PatientID + title: Patient ID + format: text + - type: metadata + value: dicom.PatientSex + title: Sex + - type: metadata + value: dicom.PatientAge + title: Age + - type: metadata + value: _count.studiescount + title: Studies + format: count + - type: metadata + value: _count.seriescount + title: Series + format: count + - type: metadata + value: _count.slicescount + title: Slices + format: count + - type: record + value: controls + + studyList: + layout: + flatten: only + group: + keys: + - dicom.StudyInstanceUID + counts: + _id: _count.slicescount + dicom.SeriesInstanceUID: _count.seriescount + # Clicking a study row hands the whole study to VolView. + navigate: + type: open + name: volview + defaultSort: + - type: metadata + value: dicom.StudyDate + dir: up + columns: + - type: image + value: thumbnail + title: Thumbnail + - type: metadata + value: dicom.PatientID + title: Patient ID + format: text + - type: metadata + value: dicom.StudyDate + title: Study Date + - type: metadata + value: dicom.StudyDescription + title: Study Description + - type: metadata + value: dicom.ModalitiesInStudy + title: Modalities + - type: metadata + value: _count.seriescount + title: Series + format: count + - type: metadata + value: _count.slicescount + title: Slices + format: count + - type: record + value: controls + + # Drill one level further when you want CT and PET as separate rows. + seriesList: + layout: + flatten: only + group: + keys: + - dicom.SeriesInstanceUID + counts: + _id: _count.slicescount + navigate: + type: open + name: volview + columns: + - type: image + value: thumbnail + title: Thumbnail + - type: metadata + value: dicom.PatientID + title: Patient ID + format: text + - type: metadata + value: dicom.Modality + title: Modality + - type: metadata + value: dicom.SeriesDescription + title: Series Description + - type: metadata + value: _count.slicescount + title: Slices + format: count + - type: record + value: controls diff --git a/e2e/seed/configs/trial/.volview_config.yaml b/e2e/seed/configs/trial/.volview_config.yaml new file mode 100644 index 0000000..43630b2 --- /dev/null +++ b/e2e/seed/configs/trial/.volview_config.yaml @@ -0,0 +1,48 @@ +# VolView app config for the trial folder (CT / PET volumes). +# +# Merged on top of girder_volview's BASE_CONFIG and inherited by every folder +# beneath this one. `seed.py seed` publishes it under two item names -- +# .volview_config.yaml and config.json -- because the folder manifest and the +# "open in VolView" link ask for different ones. +# +# Values here are constrained by VolView's zod schema (src/io/import/configJson.ts +# and src/utils/layoutParsing.ts); invalid values are dropped rather than warned +# about, so a plausible-looking guess silently does nothing: +# disabledViewTypes: only '2D', '3D', 'Oblique' +# layout view names: only axial, coronal, sagittal, volume, oblique (lowercase) +# There is no key for choosing which layout starts active -- a config can offer +# layouts, not select one. + +# CT and PET are real volumes, so re-enable the 3D view that BASE_CONFIG +# disables (it turns off both '3D' and 'Oblique'). +disabledViewTypes: + - Oblique + +layouts: + Axial Coronal Sagittal: + direction: row + items: + - axial + - direction: column + items: + - coronal + - sagittal + Quad View: + direction: column + items: + - direction: row + items: + - axial + - coronal + - direction: row + items: + - sagittal + - volume + Axial Only: + - - axial + +# Soft-tissue window, so the chest CT opens usefully framed instead of at the +# full HU range. +windowing: + width: 400 + level: 40 diff --git a/e2e/seed/configs/ultrasound/.large_image_config.yaml b/e2e/seed/configs/ultrasound/.large_image_config.yaml new file mode 100644 index 0000000..9f4825a --- /dev/null +++ b/e2e/seed/configs/ultrasound/.large_image_config.yaml @@ -0,0 +1,90 @@ +# Cine-ultrasound hierarchy view: one row per clip -> open in VolView. +# +# Grouped by SOPInstanceUID, NOT SeriesInstanceUID: each clip is one multiframe +# instance and the clips all belong to the same series, so grouping by series +# would collapse every clip into a single row. + +defaultItemList: clipList + +itemList: &plainColumns + columns: + - type: image + value: thumbnail + title: Thumbnail + - type: record + value: name + title: Name + - type: record + value: controls + title: Controls + - type: record + value: size + title: Size + +itemListDialog: *plainColumns + +namedItemLists: + clipList: + layout: + flatten: only + group: + keys: + - dicom.SOPInstanceUID + counts: + _id: _count.slicescount + navigate: + type: open + name: volview + defaultSort: + - type: metadata + value: dicom.PatientID + dir: up + columns: + - type: image + value: thumbnail + title: Thumbnail + - type: metadata + value: dicom.PatientID + title: Patient ID + format: text + - type: metadata + value: dicom.NumberOfFrames + title: Frames + - type: metadata + value: dicom.ManufacturerModelName + title: Scanner + - type: metadata + value: dicom.Manufacturer + title: Manufacturer + - type: metadata + value: dicom.StudyDate + title: Study Date + - type: record + value: controls + + # Roll the clips up by patient when a patient has several. + patientList: + layout: + flatten: only + group: + keys: dicom.PatientID + counts: + _id: _count.slicescount + dicom.SeriesInstanceUID: _count.seriescount + navigate: + type: itemList + name: clipList + columns: + - type: image + value: thumbnail + title: Thumbnail + - type: metadata + value: dicom.PatientID + title: Patient ID + format: text + - type: metadata + value: _count.seriescount + title: Clips + format: count + - type: record + value: controls diff --git a/e2e/seed/configs/ultrasound/.volview_config.yaml b/e2e/seed/configs/ultrasound/.volview_config.yaml new file mode 100644 index 0000000..c9e629f --- /dev/null +++ b/e2e/seed/configs/ultrasound/.volview_config.yaml @@ -0,0 +1,50 @@ +# VolView app config for the cine-ultrasound folder. +# +# Placed in the ultrasound folder so it overrides whatever the trial folder sets. +# See the trial config for the schema constraints these values must satisfy. +# +# A cine clip is 2D + time, so coronal and sagittal panes just reslice along the +# time axis and show noise. disabledViewTypes can't switch them off (it only +# accepts '2D', '3D', 'Oblique' -- not individual 2D orientations), so the lever +# is the layout: this folder opens straight into a single axial pane. +# +# You CAN choose which layout opens, but not with a dedicated key -- VolView +# activates whichever layout is FIRST in the map it receives (configJson.ts +# applyLayout: `layoutEntries[0][0]` -> switchToNamedLayout). +# +# Two things decide what lands first, and neither is the order written here: +# 1. Girder serializes the config response with keys sorted ALPHABETICALLY, so +# authoring order is discarded. The alphabetically first NAME wins -- hence +# the leading "1" below. +# 2. girder_volview merges this over BASE_CONFIG, which contributes its own +# "Axial Coronal Sagittal". `__all__: true` clears the base map first so +# only the layouts below are offered in the picker. + +# No volume to render from a single multiframe clip. +disabledViewTypes: + - '3D' + - Oblique + +layouts: + __all__: true + 1 Cine (single pane): + - - axial + # Kept so two clips can be compared side by side. + Axial Coronal Sagittal: + direction: row + items: + - axial + - direction: column + items: + - coronal + - sagittal + +# These clips are 8-bit, so the full range is the right window. +windowing: + width: 255 + level: 127.5 + +io: + segmentGroupExtension: seg + segmentGroupSaveFormat: nii.gz + layerExtension: layer diff --git a/e2e/seed/docker-compose.minio.yml b/e2e/seed/docker-compose.minio.yml new file mode 100644 index 0000000..2da87e4 --- /dev/null +++ b/e2e/seed/docker-compose.minio.yml @@ -0,0 +1,28 @@ +# MinIO stands in for S3 so we can exercise Girder's assetstore *import* path +# rather than plain upload. +# +# Bring it up into the EXISTING compose project so it lands on the same network +# as girder -- otherwise girder cannot resolve the `minio` hostname: +# +# docker compose -p dsa-plus -f docker-compose.minio.yml up -d +# +# Girder reaches it at http://minio:9000 (service name on the compose network). +# From the host: API on :9000, web console on :9001. + +services: + minio: + image: minio/minio + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin} + ports: + - "9000:9000" + - "9001:9001" + volumes: + - ./.minio-data:/data + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 3s + retries: 20 diff --git a/e2e/seed/manifest/series.json b/e2e/seed/manifest/series.json new file mode 100644 index 0000000..61cc89e --- /dev/null +++ b/e2e/seed/manifest/series.json @@ -0,0 +1,204 @@ +{ + "note": "Pinned IDC series. Regenerate with `seed.py select`.", + "allowed_licenses": [ + "CC BY 4.0", + "CC BY 3.0" + ], + "trial": [ + { + "patient_slot": "patient-01", + "study_slot": "study-01", + "modality_slot": "CT", + "PatientID": "ACRIN-NSCLC-FDG-PET-017", + "StudyInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.236150593949061743938807088375", + "SeriesInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.539201783671599416426540821559", + "Modality": "CT", + "SeriesDescription": "CT IMAGES", + "collection_id": "acrin_nsclc_fdg_pet", + "source_DOI": "10.7937/tcia.2019.30ilqfcl", + "license_short_name": "CC BY 3.0", + "series_size_MB": 36.462878, + "instanceCount": 69 + }, + { + "patient_slot": "patient-01", + "study_slot": "study-01", + "modality_slot": "PET", + "PatientID": "ACRIN-NSCLC-FDG-PET-017", + "StudyInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.236150593949061743938807088375", + "SeriesInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.330294229724130901703950988857", + "Modality": "PT", + "SeriesDescription": "PET NAC OSEM", + "collection_id": "acrin_nsclc_fdg_pet", + "source_DOI": "10.7937/tcia.2019.30ilqfcl", + "license_short_name": "CC BY 3.0", + "series_size_MB": 2.690166, + "instanceCount": 69 + }, + { + "patient_slot": "patient-01", + "study_slot": "study-02", + "modality_slot": "CT", + "PatientID": "ACRIN-NSCLC-FDG-PET-017", + "StudyInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.243866554226978805002562100810", + "SeriesInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.204628098576904079799739285016", + "Modality": "CT", + "SeriesDescription": "CT IMAGES", + "collection_id": "acrin_nsclc_fdg_pet", + "source_DOI": "10.7937/tcia.2019.30ilqfcl", + "license_short_name": "CC BY 3.0", + "series_size_MB": 54.431366, + "instanceCount": 103 + }, + { + "patient_slot": "patient-01", + "study_slot": "study-02", + "modality_slot": "PET", + "PatientID": "ACRIN-NSCLC-FDG-PET-017", + "StudyInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.243866554226978805002562100810", + "SeriesInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.116406849785229879410404008585", + "Modality": "PT", + "SeriesDescription": "PET NAC OSEM", + "collection_id": "acrin_nsclc_fdg_pet", + "source_DOI": "10.7937/tcia.2019.30ilqfcl", + "license_short_name": "CC BY 3.0", + "series_size_MB": 4.015208, + "instanceCount": 103 + }, + { + "patient_slot": "patient-02", + "study_slot": "study-01", + "modality_slot": "CT", + "PatientID": "ACRIN-NSCLC-FDG-PET-022", + "StudyInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.174084927553976299456785283866", + "SeriesInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.689241354457102869185568863185", + "Modality": "CT", + "SeriesDescription": "CT IMAGES", + "collection_id": "acrin_nsclc_fdg_pet", + "source_DOI": "10.7937/tcia.2019.30ilqfcl", + "license_short_name": "CC BY 3.0", + "series_size_MB": 36.462878, + "instanceCount": 69 + }, + { + "patient_slot": "patient-02", + "study_slot": "study-01", + "modality_slot": "PET", + "PatientID": "ACRIN-NSCLC-FDG-PET-022", + "StudyInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.174084927553976299456785283866", + "SeriesInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.215871834621180555229899430522", + "Modality": "PT", + "SeriesDescription": "PET NAC OSEM", + "collection_id": "acrin_nsclc_fdg_pet", + "source_DOI": "10.7937/tcia.2019.30ilqfcl", + "license_short_name": "CC BY 3.0", + "series_size_MB": 2.682006, + "instanceCount": 69 + }, + { + "patient_slot": "patient-02", + "study_slot": "study-02", + "modality_slot": "CT", + "PatientID": "ACRIN-NSCLC-FDG-PET-022", + "StudyInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.335219230286695939227482041219", + "SeriesInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.152127225372718460175886458641", + "Modality": "CT", + "SeriesDescription": "CT IMAGES", + "collection_id": "acrin_nsclc_fdg_pet", + "source_DOI": "10.7937/tcia.2019.30ilqfcl", + "license_short_name": "CC BY 3.0", + "series_size_MB": 54.431286, + "instanceCount": 103 + }, + { + "patient_slot": "patient-02", + "study_slot": "study-02", + "modality_slot": "PET", + "PatientID": "ACRIN-NSCLC-FDG-PET-022", + "StudyInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.335219230286695939227482041219", + "SeriesInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.862465022360682812778629502237", + "Modality": "PT", + "SeriesDescription": "PET NAC OSEM", + "collection_id": "acrin_nsclc_fdg_pet", + "source_DOI": "10.7937/tcia.2019.30ilqfcl", + "license_short_name": "CC BY 3.0", + "series_size_MB": 4.016434, + "instanceCount": 103 + }, + { + "patient_slot": "patient-03", + "study_slot": "study-01", + "modality_slot": "CT", + "PatientID": "ACRIN-NSCLC-FDG-PET-038", + "StudyInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.187493007984175896491435394086", + "SeriesInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.186669199088205539700786390749", + "Modality": "CT", + "SeriesDescription": "CT IMAGES", + "collection_id": "acrin_nsclc_fdg_pet", + "source_DOI": "10.7937/tcia.2019.30ilqfcl", + "license_short_name": "CC BY 3.0", + "series_size_MB": 48.085656, + "instanceCount": 91 + }, + { + "patient_slot": "patient-03", + "study_slot": "study-01", + "modality_slot": "PET", + "PatientID": "ACRIN-NSCLC-FDG-PET-038", + "StudyInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.187493007984175896491435394086", + "SeriesInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.308820346337311764119118556689", + "Modality": "PT", + "SeriesDescription": "PET NAC", + "collection_id": "acrin_nsclc_fdg_pet", + "source_DOI": "10.7937/tcia.2019.30ilqfcl", + "license_short_name": "CC BY 3.0", + "series_size_MB": 3.550996, + "instanceCount": 91 + }, + { + "patient_slot": "patient-03", + "study_slot": "study-02", + "modality_slot": "CT", + "PatientID": "ACRIN-NSCLC-FDG-PET-038", + "StudyInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.156848405313781000652346079836", + "SeriesInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.238738547360301903252273870873", + "Modality": "CT", + "SeriesDescription": "CT IMAGES", + "collection_id": "acrin_nsclc_fdg_pet", + "source_DOI": "10.7937/tcia.2019.30ilqfcl", + "license_short_name": "CC BY 3.0", + "series_size_MB": 48.08583, + "instanceCount": 91 + }, + { + "patient_slot": "patient-03", + "study_slot": "study-02", + "modality_slot": "PET", + "PatientID": "ACRIN-NSCLC-FDG-PET-038", + "StudyInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.156848405313781000652346079836", + "SeriesInstanceUID": "1.3.6.1.4.1.14519.5.2.1.7009.2403.108050635640066685905443327602", + "Modality": "PT", + "SeriesDescription": "PET NAC", + "collection_id": "acrin_nsclc_fdg_pet", + "source_DOI": "10.7937/tcia.2019.30ilqfcl", + "license_short_name": "CC BY 3.0", + "series_size_MB": 3.553914, + "instanceCount": 91 + } + ], + "ultrasound": [ + { + "clip_count": 3, + "PatientID": "LiverUS-107", + "StudyInstanceUID": "1.3.6.1.4.1.14519.5.2.1.1188.2803.132354950078663534980620689646", + "SeriesInstanceUID": "1.3.6.1.4.1.14519.5.2.1.1188.2803.156377778121234083968073390093", + "Modality": "US", + "SeriesDescription": "None", + "collection_id": "b_mode_and_ceus_liver", + "source_DOI": "10.7937/tcia.2021.v4z7-tc39", + "license_short_name": "CC BY 4.0", + "series_size_MB": 259.698808, + "instanceCount": 3 + } + ] +} diff --git a/e2e/seed/seed.py b/e2e/seed/seed.py new file mode 100644 index 0000000..2aa7656 --- /dev/null +++ b/e2e/seed/seed.py @@ -0,0 +1,1641 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "idc-index", +# "pydicom", +# "boto3", +# "girder-client", +# "pyyaml", +# "requests", +# "simpleitk", +# ] +# /// +""" +Seed a local DSA/Girder with public CC-BY DICOM via a simulated S3 import. + +Pipeline (each step is idempotent and re-runnable): + + uv run seed.py select # query IDC -> manifest/series.json (rare) + uv run seed.py fetch # download pinned series -> data/ + ATTRIBUTION.md + uv run seed.py stage # arrange into bucket layout -> MinIO + uv run seed.py seed # assetstore + import + study metadata + configs + uv run seed.py reseed # delete + recreate collections from staged data + uv run seed.py verify # assert the whole thing actually works + uv run seed.py reset # tear down the Girder side + +Data is NOT vendored: `select` pins SeriesInstanceUIDs, `fetch` downloads them +from IDC's public bucket. See ATTRIBUTION.md for the terms that ride along. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sys +import urllib.request +import zipfile +from pathlib import Path + +HERE = Path(__file__).parent.resolve() +DATA_DIR = HERE / "data" +DOWNLOAD_DIR = DATA_DIR / "downloads" +DEVELOPER_DATA_DIR = DATA_DIR / "developer" +MANIFEST_PATH = HERE / "manifest" / "series.json" +STAGED_PATH = DATA_DIR / ".staged.json" +CONFIGS_DIR = HERE / "configs" +ATTRIBUTION_PATH = HERE / "ATTRIBUTION.md" + +GIRDER_URL = os.environ.get("GIRDER_URL", "http://localhost:8080") +API_ROOT = GIRDER_URL.rstrip("/") + "/api/v1" +ADMIN_USER = os.environ.get("DSA_ADMIN_USER", "admin") +ADMIN_PASS = os.environ.get("DSA_ADMIN_PASS", "password") + +MINIO_HOST_URL = os.environ.get("MINIO_HOST_URL", "http://localhost:9000") +# What Girder (inside the compose network) uses to reach MinIO. Not the same as +# the host URL above -- the container cannot resolve "localhost:9000". +MINIO_GIRDER_URL = os.environ.get("MINIO_GIRDER_URL", "http://minio:9000") +MINIO_KEY = os.environ.get("MINIO_ROOT_USER", "minioadmin") +MINIO_SECRET = os.environ.get("MINIO_ROOT_PASSWORD", "minioadmin") +BUCKET = os.environ.get("DEVKIT_BUCKET", "dsa-devkit") + +VOLVIEW_CONFIG_NAME = ".volview_config.yaml" + +# VolView validates its config with zod and silently drops values that don't +# match, so an invalid-but-plausible entry looks like it applied and does +# nothing. Mirrored from src/io/import/configJson.ts and utils/layoutParsing.ts. +VALID_DISABLED_VIEW_TYPES = {"2D", "3D", "Oblique"} +VALID_LAYOUT_VIEWS = {"axial", "coronal", "sagittal", "volume", "oblique"} + +# VolView opens whichever layout comes first in the map it receives, and Girder +# serializes config keys alphabetically -- so the layout that opens is decided by +# NAME, not by the order written in the YAML. Pin the intent so a rename can't +# silently change which layout users land in. +EXPECTED_ACTIVE_LAYOUT = { + "patients": "Axial Coronal Sagittal", + "prostate": "Axial Coronal Sagittal", + "ultrasound": "1 Cine (single pane)", +} + +UNFILTERED_COLLECTION_NAME = "Trial" +FILTERED_COLLECTION_NAME = "Trial (Large Image Filter)" +DEVELOPER_COLLECTION_NAME = "Developer" +COLLECTION_NAMES = ( + UNFILTERED_COLLECTION_NAME, + FILTERED_COLLECTION_NAME, + DEVELOPER_COLLECTION_NAME, +) +TRIAL_COLLECTION_NAMES = ( + UNFILTERED_COLLECTION_NAME, + FILTERED_COLLECTION_NAME, +) +ASSETSTORE_NAME = "Devkit MinIO" +ASSETSTORE_TYPE_S3 = 2 # girder.constants.AssetstoreType.S3 + +DEVELOPER_DOWNLOADS = { + "prostate": { + "url": "https://data.kitware.com/api/v1/file/63527c7311dab8142820a339/download", + "name": "MRI-PROSTATEx-0004.zip", + "sha512": ( + "4f5c5e8a8230e950ae6dd280f3128ef62ac5d44e5e49c6c1f2d3e07482df4d7b" + "e0bf55986f8f397b0f9671c9a9b75fc4fbf4e93fbbc889a701939f3780d3b2b3" + ), + }, + "prostate_seg": { + "url": "https://data.kitware.com/api/v1/file/692f13ed80eaefe49a4abb72/download", + "name": "prostate-total.seg.nii.gz", + "sha512": ( + "bb2919662086e670bf4666f803a27f6ffd95cc5461e3381cb0d4e50df7e62c864" + "9ec6a2a2bb6cd726d73542db71831280408251139d8b24dde5e28bd22886459" + ), + }, + "fetus": { + "url": "https://data.kitware.com/api/v1/file/635679c311dab8142820a4f5/download", + "name": "3DUS-Fetus.mha", + "sha512": ( + "93342dfe499ac855a4e51ed9bf16358fe265a17ca08b03f04a0cb43538afc5fdb" + "6633d9ef4e629de93845c7a859f91993a452152c788104fcb04652358cd3ead" + ), + }, +} + +# Only these are safe to redistribute. IDC licenses per SERIES, not per +# collection, so this is filtered on every pick rather than assumed. +ALLOWED_LICENSES = ("CC BY 4.0", "CC BY 3.0") + +# ACRIN 6668: FDG-PET/CT of NSCLC with an explicit baseline + post-treatment +# design, so patients genuinely have repeat CT+PET studies. CMB-LCA looks like an +# obvious choice but only 13 of its studies have both CT and PT, and no patient +# has more than one -- so a 3x2 trial hierarchy is impossible there. +TRIAL_COLLECTION = "acrin_nsclc_fdg_pet" + +# Real cine: verified 35-41 frame color loops from GE scanners. The CMB +# ultrasound is single-frame stills and prostate_mri_us_biopsy is a 3D volume +# stack (multiframe, but not a temporal loop), so neither is what we want here. +US_COLLECTION = "b_mode_and_ceus_liver" + +N_PATIENTS = 3 +N_STUDIES = 2 +N_CLIPS = 3 + +# The e2e "small tier": a couple of real multi-file series uploaded directly +# (no MinIO) into a test folder — one study whose CT+PET pair exercises +# PET-over-CT layering, plus a second patient's CT so filtering has something +# to exclude. (patient_slot, study_slot, modality_slot) of manifest entries. +SMALL_TIER_SLOTS = { + ("patient-01", "study-01", "CT"), + ("patient-01", "study-01", "PET"), + ("patient-02", "study-01", "CT"), +} + +# Skip scouts/topograms and keep each series small enough to seed quickly. +CT_MIN_INSTANCES = 40 +CT_SIZE_MB = (5, 90) +PT_SIZE_MB = (2, 60) +US_MAX_SERIES_MB = 400 +# Instance size is what separates a cine loop from a still in this collection: +# a multi-frame loop is tens of MB, a single-frame still is well under one. +US_MIN_MB_PER_INSTANCE = 20 + +# DICOM tags retained in the staging plan. girder_volview reads the instance +# tags into meta.dicom.* during import; the plan also carries study-level fields +# derived from the selected series. +META_TAGS = [ + "PatientID", + "PatientName", + "PatientSex", + "PatientAge", + "StudyInstanceUID", + "StudyDescription", + "StudyDate", + "SeriesInstanceUID", + "SeriesDescription", + "SeriesNumber", + "Modality", + "Manufacturer", + "ManufacturerModelName", + "NumberOfFrames", + "ModalitiesInStudy", + # Cine clips share a SeriesInstanceUID, so the ultrasound view groups on this + # instead to get one row per clip. + "SOPInstanceUID", +] + + +def log(msg: str) -> None: + print(msg, flush=True) + + +def die(msg: str) -> "None": + print(f"ERROR: {msg}", file=sys.stderr, flush=True) + raise SystemExit(1) + + +def stack_up(timeout: float = 3.0) -> bool: + try: + with urllib.request.urlopen(API_ROOT + "/system/version", timeout=timeout) as r: + return r.status == 200 + except Exception: + return False + + +def minio_up(timeout: float = 3.0) -> bool: + # MinIO answers /minio/health/live without credentials. + try: + url = MINIO_HOST_URL.rstrip("/") + "/minio/health/live" + with urllib.request.urlopen(url, timeout=timeout) as r: + return r.status == 200 + except Exception: + return False + + +def read_manifest() -> dict: + if not MANIFEST_PATH.exists(): + die(f"No manifest at {MANIFEST_PATH}. Run `seed.py select` first.") + return json.loads(MANIFEST_PATH.read_text()) + + +def girder_client(): + from girder_client import GirderClient + + if not stack_up(): + die(f"Girder not reachable at {API_ROOT}. Is the dsa-plus stack up?") + gc = GirderClient(apiUrl=API_ROOT) + gc.authenticate(ADMIN_USER, ADMIN_PASS) + return gc + + +def s3_client(): + import boto3 + + return boto3.client( + "s3", + endpoint_url=MINIO_HOST_URL, + aws_access_key_id=MINIO_KEY, + aws_secret_access_key=MINIO_SECRET, + region_name="us-east-1", + ) + + +def pick_trial_series(idx): + """3 patients x 2 studies, each study contributing one CT and one PET series. + + Returns a list of pick dicts. Pure w.r.t. the dataframe -- no I/O. + """ + pool = idx[ + (idx.collection_id == TRIAL_COLLECTION) + & (idx.license_short_name.isin(ALLOWED_LICENSES)) + ] + ct = pool[ + (pool.Modality == "CT") + & (pool.instanceCount >= CT_MIN_INSTANCES) + & pool.series_size_MB.between(*CT_SIZE_MB) + ] + pt = pool[ + (pool.Modality == "PT") + & (pool.instanceCount >= CT_MIN_INSTANCES) + & pool.series_size_MB.between(*PT_SIZE_MB) + ] + + both = set(ct.StudyInstanceUID) & set(pt.StudyInstanceUID) + if not both: + die(f"No {TRIAL_COLLECTION} study has both a CT and a PT series in bounds.") + + pairs = ( + pool[pool.StudyInstanceUID.isin(both)][ + ["PatientID", "StudyInstanceUID", "StudyDate"] + ] + .drop_duplicates(subset=["PatientID", "StudyInstanceUID"]) + .sort_values(["PatientID", "StudyDate"]) + ) + longitudinal = pairs.groupby("PatientID").filter(lambda g: len(g) >= N_STUDIES) + if longitudinal.empty: + die(f"No {TRIAL_COLLECTION} patient has >= {N_STUDIES} CT+PET studies.") + + picks = [] + patients = list(longitudinal.groupby("PatientID"))[:N_PATIENTS] + if len(patients) < N_PATIENTS: + log( + f" warning: only {len(patients)} qualifying patients (wanted {N_PATIENTS})" + ) + + for p_i, (_patient_id, group) in enumerate(patients, start=1): + studies = list(group.StudyInstanceUID)[:N_STUDIES] + for s_i, study_uid in enumerate(studies, start=1): + for modality, source in (("CT", ct), ("PET", pt)): + rows = source[source.StudyInstanceUID == study_uid] + if rows.empty: + continue + # Smallest qualifying series keeps the seed quick. + row = rows.sort_values("series_size_MB").iloc[0] + picks.append( + { + "patient_slot": f"patient-{p_i:02d}", + "study_slot": f"study-{s_i:02d}", + "modality_slot": modality, + "PatientID": str(row.PatientID), + "StudyInstanceUID": str(row.StudyInstanceUID), + "SeriesInstanceUID": str(row.SeriesInstanceUID), + "Modality": str(row.Modality), + "SeriesDescription": str(row.SeriesDescription), + "collection_id": str(row.collection_id), + "source_DOI": str(row.source_DOI), + "license_short_name": str(row.license_short_name), + "series_size_MB": float(row.series_size_MB), + "instanceCount": int(row.instanceCount), + } + ) + return picks + + +def pick_us_series(idx): + """One cine-ultrasound series; each of its instances becomes a clip. + + The index has no NumberOfFrames column, so cine-ness cannot be confirmed + here -- `fetch` proves it with pydicom and fails loudly if a pick turns out + to be a still. The proxy is average instance size: these series mix cine + loops (tens of MB) with single-frame stills (well under 1 MB), so a large + mean instance means the series is loops. Among those, take the smallest + total, since downloads are whole-series. + """ + us = idx[ + (idx.collection_id == US_COLLECTION) + & (idx.Modality == "US") + & idx.license_short_name.isin(ALLOWED_LICENSES) + & (idx.instanceCount >= N_CLIPS) + & (idx.series_size_MB <= US_MAX_SERIES_MB) + ].copy() + + us = us[us.series_size_MB / us.instanceCount >= US_MIN_MB_PER_INSTANCE] + if us.empty: + die( + f"No {US_COLLECTION} series with >= {N_CLIPS} instances averaging " + f">= {US_MIN_MB_PER_INSTANCE} MB under {US_MAX_SERIES_MB} MB total." + ) + + row = us.sort_values("series_size_MB").iloc[0] + + return [ + { + "clip_count": N_CLIPS, + "PatientID": str(row.PatientID), + "StudyInstanceUID": str(row.StudyInstanceUID), + "SeriesInstanceUID": str(row.SeriesInstanceUID), + "Modality": "US", + "SeriesDescription": str(row.SeriesDescription), + "collection_id": str(row.collection_id), + "source_DOI": str(row.source_DOI), + "license_short_name": str(row.license_short_name), + "series_size_MB": float(row.series_size_MB), + "instanceCount": int(row.instanceCount), + } + ] + + +def cmd_select(args) -> None: + from idc_index import IDCClient + + log("Loading the IDC index (first run downloads ~77 MB and takes a moment)...") + client = IDCClient() + idx = client.index + log(f" index: {len(idx):,} series") + + log(f"Selecting CT+PET trial series from {TRIAL_COLLECTION}...") + trial = pick_trial_series(idx) + n_patients = len({p["patient_slot"] for p in trial}) + log(f" {len(trial)} series across {n_patients} patients") + + log("Selecting a cine-ultrasound series...") + ultrasound = pick_us_series(idx) + for p in ultrasound: + log( + f" {p['PatientID']}: {p['instanceCount']} instances, " + f"{p['series_size_MB']:.0f} MB -> {p['clip_count']} clips" + ) + + bad = [ + p for p in trial + ultrasound if p["license_short_name"] not in ALLOWED_LICENSES + ] + if bad: + die(f"{len(bad)} picks are not CC-BY -- refusing to write the manifest.") + + manifest = { + "note": "Pinned IDC series. Regenerate with `seed.py select`.", + "allowed_licenses": list(ALLOWED_LICENSES), + "trial": trial, + "ultrasound": ultrasound, + } + MANIFEST_PATH.parent.mkdir(parents=True, exist_ok=True) + MANIFEST_PATH.write_text(json.dumps(manifest, indent=2) + "\n") + total = sum(p["series_size_MB"] for p in trial + ultrasound) + n_series = len(trial) + len(ultrasound) + log(f"\nWrote {MANIFEST_PATH} ({n_series} series, ~{total:.0f} MB)") + + +def series_dir(series_uid: str) -> Path: + return DATA_DIR / series_uid + + +def small_tier_picks(manifest: dict) -> list[dict]: + picks = [ + p + for p in manifest["trial"] + if (p["patient_slot"], p["study_slot"], p["modality_slot"]) in SMALL_TIER_SLOTS + ] + if len(picks) != len(SMALL_TIER_SLOTS): + die( + f"Manifest lacks the small-tier series {sorted(SMALL_TIER_SLOTS)}; " + "re-run `seed.py select`?" + ) + return picks + + +def sha512(path: Path) -> str: + digest = hashlib.sha512() + with open(path, "rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def download_verified(source: dict, force: bool = False) -> Path: + """Download a pinned example and reject incomplete or changed content.""" + DOWNLOAD_DIR.mkdir(parents=True, exist_ok=True) + destination = DOWNLOAD_DIR / source["name"] + if destination.exists() and not force and sha512(destination) == source["sha512"]: + log(f" {source['name']}: already present") + return destination + + partial = destination.with_suffix(destination.suffix + ".part") + partial.unlink(missing_ok=True) + log(f" downloading {source['name']}...") + try: + urllib.request.urlretrieve(source["url"], partial) + actual = sha512(partial) + if actual != source["sha512"]: + die( + f"SHA-512 mismatch for {source['name']}: " + f"expected {source['sha512']}, got {actual}" + ) + partial.replace(destination) + finally: + partial.unlink(missing_ok=True) + return destination + + +def add_segmentation_metadata(image, segments: list[tuple[int, str, str]]) -> None: + """Add the core 3D Slicer segmentation fields understood by VolView.""" + fields = { + "Segmentation_ContainedRepresentationNames": "Binary labelmap|", + "Segmentation_MasterRepresentation": "Binary labelmap", + } + for index, (label, name, color) in enumerate(segments): + fields.update( + { + f"Segment{index}_ID": name.lower().replace(" ", "_"), + f"Segment{index}_Name": name, + f"Segment{index}_Color": color, + f"Segment{index}_LabelValue": str(label), + f"Segment{index}_Layer": "0", + } + ) + for key, value in fields.items(): + image.SetMetaData(key, value) + + +def prepare_developer_examples(force: bool = False) -> None: + """Fetch VolView's prostate/fetus examples and create associated NRRDs.""" + import SimpleITK as sitk + + log("\nFetching developer examples...") + downloads = { + key: download_verified(source, force) + for key, source in DEVELOPER_DOWNLOADS.items() + } + + prostate_dir = DEVELOPER_DATA_DIR / "prostate" + prostate_dicom_dir = prostate_dir / "dicom" + fetus_dir = DEVELOPER_DATA_DIR / "fetus" + prostate_dicom_dir.mkdir(parents=True, exist_ok=True) + fetus_dir.mkdir(parents=True, exist_ok=True) + + with zipfile.ZipFile(downloads["prostate"]) as archive: + dicom_members = [ + member for member in archive.infolist() if member.filename.endswith(".dcm") + ] + if not dicom_members: + die(f"{downloads['prostate'].name} contains no DICOM instances") + for member in dicom_members: + destination = prostate_dicom_dir / Path(member.filename).name + if destination.exists() and not force: + continue + with archive.open(member) as source, open(destination, "wb") as output: + while chunk := source.read(1024 * 1024): + output.write(chunk) + + prostate_seg = prostate_dir / "5.seg.total-segmentator.nrrd" + image = sitk.ReadImage(str(downloads["prostate_seg"])) + stats = sitk.LabelShapeStatisticsImageFilter() + stats.Execute(image) + labels = sorted(int(label) for label in stats.GetLabels()) + palette = ("0.9 0.2 0.2", "0.2 0.7 0.9", "0.3 0.8 0.3", "0.9 0.7 0.2") + add_segmentation_metadata( + image, + [ + (label, f"Prostate label {label}", palette[index % len(palette)]) + for index, label in enumerate(labels) + ], + ) + sitk.WriteImage(image, str(prostate_seg), True) + + fetus_image = fetus_dir / "fetus.mha" + if force or not fetus_image.exists(): + fetus_image.write_bytes(downloads["fetus"].read_bytes()) + + fetus_seg = fetus_dir / "fetus.seg.nrrd" + image = sitk.ReadImage(str(fetus_image)) + segmentation = sitk.Cast(sitk.OtsuThreshold(image, 0, 1), sitk.sitkUInt8) + add_segmentation_metadata(segmentation, [(1, "Fetus foreground", "0.95 0.65 0.2")]) + sitk.WriteImage(segmentation, str(fetus_seg), True) + + log(f" prostate: {len(list(prostate_dicom_dir.glob('*.dcm')))} DICOM slices") + log(f" prostate segmentation: {prostate_seg.name}") + log(f" fetus image + segmentation: {fetus_image.name}, {fetus_seg.name}") + + +def cmd_fetch(args) -> None: + from idc_index import IDCClient + + manifest = read_manifest() + small = getattr(args, "small", False) + picks = ( + small_tier_picks(manifest) + if small + else manifest["trial"] + manifest["ultrasound"] + ) + + wanted = [p["SeriesInstanceUID"] for p in picks] + missing = [uid for uid in wanted if not list(series_dir(uid).glob("*.dcm"))] + + DATA_DIR.mkdir(parents=True, exist_ok=True) + if missing and not args.force: + log( + f"Downloading {len(missing)} series from IDC " + f"({len(wanted) - len(missing)} already present)..." + ) + elif args.force: + missing = wanted + log(f"Re-downloading all {len(missing)} series (--force)...") + else: + log("All pinned series already present.") + + if missing: + client = IDCClient() + client.download_from_selection( + seriesInstanceUID=missing, + downloadDir=str(DATA_DIR), + dirTemplate="%SeriesInstanceUID", + dry_run=False, + show_progress_bar=True, + ) + + verify_fetch(picks) + if small: + # ATTRIBUTION.md documents the FULL pinned set; regenerating it from a + # three-series subset would shrink the committed citations. + log("Skipping ATTRIBUTION.md regeneration (--small subset).") + else: + prepare_developer_examples(args.force) + write_attribution(picks) + + +def verify_fetch(picks: list[dict]) -> None: + """Assert every series landed, and that US picks are genuinely multiframe.""" + import pydicom + + log("\nVerifying downloads...") + problems = [] + for p in picks: + files = sorted(series_dir(p["SeriesInstanceUID"]).glob("*.dcm")) + if not files: + problems.append(f"{p['SeriesInstanceUID']}: no files downloaded") + continue + if p["license_short_name"] not in ALLOWED_LICENSES: + problems.append( + f"{p['SeriesInstanceUID']}: license {p['license_short_name']}" + ) + if p["Modality"] == "US": + # Every instance we intend to stage as a clip must really be cine. + for i, path in enumerate(files[: p["clip_count"]], start=1): + ds = pydicom.dcmread(path, stop_before_pixels=True) + frames = int(getattr(ds, "NumberOfFrames", 1) or 1) + if frames <= 1: + problems.append( + f"{path.name}: US but NumberOfFrames={frames} " + "(not a cine loop -- re-run `select`)" + ) + else: + log(f" clip-{i:02d}: {frames} frames OK") + else: + slot = f"{p['patient_slot']}/{p['study_slot']}/{p['modality_slot']}" + log(f" {slot}: {len(files)} slices") + + if problems: + for problem in problems: + print(f" FAIL {problem}", file=sys.stderr) + die(f"{len(problems)} series failed verification.") + log("All series verified.") + + +def clean_citation(text: str) -> str: + """IDC returns APA citations as HTML; flatten them for markdown.""" + import html + import re + + return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", "", str(text)))).strip() + + +def write_attribution(picks: list[dict]) -> None: + """Regenerate ATTRIBUTION.md. Citations come from IDC, not a hand-kept list.""" + from idc_index import IDCClient + + log("\nGenerating ATTRIBUTION.md...") + client = IDCClient() + try: + citations = client.citations_from_selection( + seriesInstanceUID=[p["SeriesInstanceUID"] for p in picks] + ) + except Exception as exc: # network/API hiccup shouldn't lose the download + log(f" warning: could not fetch citations ({exc}); falling back to DOIs") + citations = sorted({p["source_DOI"] for p in picks}) + + if isinstance(citations, (list, tuple)): + citations_text = "\n".join(f"- {clean_citation(c)}" for c in citations) + else: + citations_text = clean_citation(str(citations)) + + dois = sorted({p["source_DOI"] for p in picks}) + licenses = sorted({p["license_short_name"] for p in picks}) + + ATTRIBUTION_PATH.write_text( + f"""# Attribution + +The DICOM data this devkit downloads comes from the NCI Imaging Data Commons +(IDC) and is redistributed under {" / ".join(licenses)}. No imaging data is +stored in this repository -- `seed.py fetch` pulls it from IDC's public bucket. + +## Datasets used + +{citations_text} +- Litjens, G., Debats, O., Barentsz, J., Karssemeijer, N., & Huisman, H. + (2017). *SPIE-AAPM PROSTATEx Challenge Data* (Version 2) [Dataset]. The + Cancer Imaging Archive. https://doi.org/10.7937/K9TCIA.2017.MURS5CL + +IDC selection DOIs: {", ".join(dois)} + +## VolView developer examples + +The prostate subset and fetal ultrasound volume are pinned by SHA-512 and +downloaded from Kitware's public VolView example-data folder: + +- `MRI-PROSTATEx-0004.zip` +- `prostate-total.seg.nii.gz` (converted locally to `.seg.nrrd`) +- `3DUS-Fetus.mha` + +The fetal segmentation is generated locally with Otsu thresholding; it is not +a clinical annotation. + +## IDC + +Fedorov, A., Longabaugh, W. J. R., Pot, D., et al. *National Cancer Institute +Imaging Data Commons: Toward Transparency, Reproducibility, and Scalability in +Imaging Artificial Intelligence.* RadioGraphics (2023). +https://doi.org/10.1148/rg.230180 + +## Terms that travel with this data + +Per the TCIA Data Usage Policy +(https://www.cancerimagingarchive.net/data-usage-policies-and-restrictions/): + +- Attribute each individual dataset used, and link to that policy. +- Pass this same obligation on to downstream users. +- Do not attempt to identify or contact the individuals these images came from. + +Regenerate this file with `uv run seed.py fetch`. +""", + ) + log(f" wrote {ATTRIBUTION_PATH}") + + +def sorted_slices(files: list[Path]) -> list[Path]: + """Order a series by InstanceNumber so subsampling stays anatomically sane.""" + import pydicom + + def key(path: Path): + try: + ds = pydicom.dcmread(path, stop_before_pixels=True) + return (int(getattr(ds, "InstanceNumber", 0) or 0), path.name) + except Exception: + return (0, path.name) + + return sorted(files, key=key) + + +def subsample(files: list[Path], limit: int) -> list[Path]: + """Take at most `limit` files at a uniform stride. + + Uniform stride keeps slice spacing constant, so the volume stays valid -- + just coarser. Taking the first N would truncate the anatomy instead. + """ + if limit <= 0 or len(files) <= limit: + return files + stride = len(files) / limit + return [files[int(i * stride)] for i in range(limit)] + + +def file_metadata(path: Path) -> dict: + """Retain selected DICOM tags in the staging plan.""" + import pydicom + + ds = pydicom.dcmread(path, stop_before_pixels=True) + meta = {} + for tag in META_TAGS: + value = getattr(ds, tag, None) + if value is None or value == "": + continue + # Only true multi-valued elements become lists. Testing for __iter__ + # instead would explode a PersonName into a list of characters, since + # pydicom's PersonName iterates per character. + if isinstance(value, pydicom.multival.MultiValue): + value = [str(v) for v in value] + else: + value = str(value) + meta[tag] = value + return meta + + +def build_staging_plan(manifest: dict, max_slices: int) -> dict: + """manifest + local files -> the exact set of objects to upload. + + Metadata is captured per object rather than per series: the cine clips all + share one SeriesInstanceUID and are told apart by SOPInstanceUID, which the + ultrasound view groups on. + """ + objects = [] + + # ModalitiesInStudy is a query-level attribute, absent from the instances + # themselves, so the study view's Modalities column would render empty. + # Derive it from what we actually staged into each study. + study_modalities: dict[str, set] = {} + for p in manifest["trial"]: + study_modalities.setdefault(p["StudyInstanceUID"], set()).add(p["Modality"]) + + for p in manifest["trial"]: + uid = p["SeriesInstanceUID"] + files = sorted_slices(list(series_dir(uid).glob("*.dcm"))) + if not files: + die(f"Missing local files for {uid}. Run `seed.py fetch`.") + prefix = f"trial/{p['patient_slot']}/{p['study_slot']}/{p['modality_slot']}" + modalities = sorted(study_modalities[p["StudyInstanceUID"]]) + for i, path in enumerate(subsample(files, max_slices)): + meta = file_metadata(path) + meta["ModalitiesInStudy"] = modalities + objects.append( + {"key": f"{prefix}/{i:04d}.dcm", "local": str(path), "meta": meta} + ) + + for p in manifest["ultrasound"]: + uid = p["SeriesInstanceUID"] + files = sorted(series_dir(uid).glob("*.dcm")) + if not files: + die(f"Missing local files for {uid}. Run `seed.py fetch`.") + for i, path in enumerate(files[: p["clip_count"]], start=1): + objects.append( + { + "key": f"ultrasound/clip-{i:02d}.dcm", + "local": str(path), + "meta": file_metadata(path), + "content_type": "application/dicom", + } + ) + + developer_files = { + "developer/prostate/5.seg.total-segmentator.nrrd": ( + DEVELOPER_DATA_DIR / "prostate" / "5.seg.total-segmentator.nrrd", + "application/octet-stream", + ), + "developer/fetus/fetus.mha": ( + DEVELOPER_DATA_DIR / "fetus" / "fetus.mha", + "application/octet-stream", + ), + "developer/fetus/fetus.seg.nrrd": ( + DEVELOPER_DATA_DIR / "fetus" / "fetus.seg.nrrd", + "application/octet-stream", + ), + } + prostate_dicom = sorted((DEVELOPER_DATA_DIR / "prostate" / "dicom").glob("*.dcm")) + if not prostate_dicom: + die("Developer examples are missing. Run `seed.py fetch`.") + for path in prostate_dicom: + developer_files[f"developer/prostate/dicom/{path.name}"] = ( + path, + "application/dicom", + ) + for key, (path, content_type) in developer_files.items(): + if not path.exists(): + die(f"Developer example {path} is missing. Run `seed.py fetch`.") + objects.append( + { + "key": key, + "local": str(path), + "meta": {}, + "content_type": content_type, + } + ) + + return {"bucket": BUCKET, "objects": objects} + + +def cmd_stage(args) -> None: + if not minio_up(): + die( + f"MinIO not reachable at {MINIO_HOST_URL}.\n" + " Start it with: docker compose -p dsa-plus " + f"-f {HERE / 'docker-compose.minio.yml'} up -d" + ) + + manifest = read_manifest() + log(f"Building staging plan (max {args.max_slices} slices per series)...") + plan = build_staging_plan(manifest, args.max_slices) + log(f" {len(plan['objects'])} objects") + + s3 = s3_client() + existing_buckets = {b["Name"] for b in s3.list_buckets().get("Buckets", [])} + if BUCKET not in existing_buckets: + log(f"Creating bucket {BUCKET}") + s3.create_bucket(Bucket=BUCKET) + + already = set() + paginator = s3.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=BUCKET): + already.update(o["Key"] for o in page.get("Contents", [])) + + uploaded = 0 + for obj in plan["objects"]: + if obj["key"] in already and not args.force: + continue + s3.upload_file( + obj["local"], + BUCKET, + obj["key"], + ExtraArgs={"ContentType": obj.get("content_type", "application/dicom")}, + ) + uploaded += 1 + if uploaded % 25 == 0: + log(f" uploaded {uploaded}...") + + STAGED_PATH.write_text(json.dumps(plan, indent=2) + "\n") + already = len(plan["objects"]) - uploaded + log(f"Uploaded {uploaded} objects ({already} already present).") + log(f"Wrote staging plan to {STAGED_PATH}") + + +def get_setting(gc, key: str): + return gc.get("system/setting", parameters={"key": key}) + + +def set_setting(gc, key: str, value) -> None: + gc.put("system/setting", parameters={"key": key, "value": json.dumps(value)}) + + +def ensure_collection(gc, name: str) -> dict: + for coll in gc.listCollection(): + if coll["name"] == name: + return coll + return gc.post("collection", parameters={"name": name, "public": "true"}) + + +def ensure_folder(gc, parent_id: str, parent_type: str, name: str) -> dict: + return gc.createFolder( + parent_id, name, parentType=parent_type, reuseExisting=True, public=True + ) + + +def ensure_assetstore(gc) -> dict: + for store in gc.get("assetstore"): + if store["name"] == ASSETSTORE_NAME: + return store + log(f"Creating S3 assetstore {ASSETSTORE_NAME!r} -> {MINIO_GIRDER_URL}/{BUCKET}") + # Girder probes the bucket with a real put_object here, so the bucket must + # already exist -- `stage` runs first. + return gc.post( + "assetstore", + parameters={ + "type": ASSETSTORE_TYPE_S3, + "name": ASSETSTORE_NAME, + "bucket": BUCKET, + "prefix": "", + "accessKeyId": MINIO_KEY, + "secret": MINIO_SECRET, + "service": MINIO_GIRDER_URL, + "region": "us-east-1", + }, + ) + + +def import_prefix( + gc, + assetstore_id: str, + prefix: str, + folder_id: str, + file_include_regex: str = r".*\.dcm$", +) -> None: + log(f"Importing s3://{BUCKET}/{prefix} -> folder {folder_id}") + gc.post( + f"assetstore/{assetstore_id}/import", + parameters={ + "importPath": prefix, + "destinationId": folder_id, + "destinationType": "folder", + "progress": True, + # Anchored: Girder matches this with re.match against the basename, + # not re.search, so a bare r"\.dcm$" silently imports nothing. + "fileIncludeRegex": file_include_regex, + }, + ) + + +def folder_path_index(gc, root_id: str) -> dict: + """Map relative folder path -> folder id, walking down from root.""" + index = {(): root_id} + + def walk(folder_id: str, parts: tuple): + for child in gc.listFolder(folder_id, parentFolderType="folder"): + key = parts + (child["name"],) + index[key] = child["_id"] + walk(child["_id"], key) + + walk(root_id, ()) + return index + + +def add_derived_dicom_metadata(gc, item_id: str, metadata: dict) -> None: + """Merge seed-derived fields with girder_volview's parsed DICOM metadata.""" + item = gc.getItem(item_id) + dicom = item.get("meta", {}).get("dicom", {}) + gc.addMetadataToItem(item_id, {"dicom": {**dicom, **metadata}}) + + +def apply_study_metadata(gc, plan: dict, roots: dict) -> int: + """Add study-level fields that are absent from individual DICOM instances. + + girder_volview populates the instance tags synchronously from + ``model.file.save.after`` for both uploads and asset-store imports. + ``ModalitiesInStudy`` is derived from all series staged for a study, so the + seed adds only that enrichment without replacing the parsed tags. + """ + log("Applying derived study metadata to imported items...") + + # folder id -> {item name: item id}, filled lazily. + item_cache: dict[str, dict] = {} + path_cache: dict[str, dict] = {} + applied, orphaned = 0, [] + + for obj in plan["objects"]: + derived = { + key: obj["meta"][key] + for key in ("ModalitiesInStudy",) + if key in obj["meta"] + } + if not derived: + continue + + parts = obj["key"].split("/") + top, rel_dirs, filename = parts[0], parts[1:-1], parts[-1] + root_id = roots.get(top) + if root_id is None: + continue + + if top not in path_cache: + path_cache[top] = folder_path_index(gc, root_id) + folder_id = path_cache[top].get(tuple(rel_dirs)) + if folder_id is None: + orphaned.append(obj["key"]) + continue + + if folder_id not in item_cache: + item_cache[folder_id] = { + item["name"]: item["_id"] for item in gc.listItem(folder_id) + } + item_id = item_cache[folder_id].get(filename) + if item_id is None: + orphaned.append(obj["key"]) + continue + + add_derived_dicom_metadata(gc, item_id, derived) + applied += 1 + if applied % 50 == 0: + log(f" {applied}...") + + if orphaned: + log(f" warning: {len(orphaned)} staged objects had no matching item") + for key in orphaned[:5]: + log(f" {key}") + log(f" applied derived metadata to {applied} items") + return applied + + +def upload_config(gc, folder_id: str, path: Path, item_name: str | None = None) -> None: + item = gc.createItem(folder_id, item_name or path.name, reuseExisting=True) + # Replace rather than append, so re-seeding picks up edited configs. + for existing in gc.listFile(item["_id"]): + gc.delete(f"file/{existing['_id']}") + with open(path, "rb") as fh: + gc.uploadFile( + parentId=item["_id"], + stream=fh, + name=item_name or path.name, + size=path.stat().st_size, + parentType="item", + mimeType="application/x-yaml", + ) + + +def cmd_seed(args) -> None: + if not STAGED_PATH.exists(): + die(f"No staging plan at {STAGED_PATH}. Run `seed.py stage` first.") + plan = json.loads(STAGED_PATH.read_text()) + + gc = girder_client() + + # The import path fires model.file.save per file, which walks into + # large_image's DICOM adjacency scan -- an O(n^2) blowup on large series. + # Off during import, restored after. + previous_auto_set = get_setting(gc, "large_image.auto_set") + log(f"Disabling large_image.auto_set during import (was {previous_auto_set!r})") + set_setting(gc, "large_image.auto_set", False) + + collections = {} + try: + assetstore = ensure_assetstore(gc) + + for collection_name in TRIAL_COLLECTION_NAMES: + collection = ensure_collection(gc, collection_name) + collections[collection_name] = collection + log(f"Collection {collection_name!r}: {collection['_id']}") + patients = ensure_folder(gc, collection["_id"], "collection", "patients") + ultrasound = ensure_folder( + gc, collection["_id"], "collection", "ultrasound" + ) + import_prefix(gc, assetstore["_id"], "trial/", patients["_id"]) + import_prefix(gc, assetstore["_id"], "ultrasound/", ultrasound["_id"]) + apply_study_metadata(gc, plan, {"trial": patients["_id"]}) + + trial_config_dir = CONFIGS_DIR / "trial" + upload_config(gc, patients["_id"], trial_config_dir / VOLVIEW_CONFIG_NAME) + log(f" {collection_name}/patients/{VOLVIEW_CONFIG_NAME}") + if collection_name == FILTERED_COLLECTION_NAME: + large_image_config = trial_config_dir / ".large_image_config.yaml" + upload_config(gc, patients["_id"], large_image_config) + log(f" {collection_name}/patients/{large_image_config.name}") + ultrasound_config = CONFIGS_DIR / "ultrasound" / VOLVIEW_CONFIG_NAME + upload_config(gc, ultrasound["_id"], ultrasound_config) + log(f" {collection_name}/ultrasound/{VOLVIEW_CONFIG_NAME}") + if collection_name == FILTERED_COLLECTION_NAME: + ultrasound_filter = ( + CONFIGS_DIR / "ultrasound" / ".large_image_config.yaml" + ) + upload_config(gc, ultrasound["_id"], ultrasound_filter) + log(f" {collection_name}/ultrasound/{ultrasound_filter.name}") + + developer = ensure_collection(gc, DEVELOPER_COLLECTION_NAME) + collections[DEVELOPER_COLLECTION_NAME] = developer + log(f"Collection {DEVELOPER_COLLECTION_NAME!r}: {developer['_id']}") + developer_roots = { + name: ensure_folder(gc, developer["_id"], "collection", name)["_id"] + for name in ("prostate", "fetus", "ultrasound") + } + image_regex = r".*\.(dcm|mha|nrrd)$" + import_prefix( + gc, + assetstore["_id"], + "developer/prostate/", + developer_roots["prostate"], + image_regex, + ) + import_prefix( + gc, + assetstore["_id"], + "developer/fetus/", + developer_roots["fetus"], + image_regex, + ) + import_prefix( + gc, + assetstore["_id"], + "ultrasound/", + developer_roots["ultrasound"], + ) + developer_config = CONFIGS_DIR / "developer" / VOLVIEW_CONFIG_NAME + for name in ("prostate", "fetus"): + upload_config(gc, developer_roots[name], developer_config) + log(f" {DEVELOPER_COLLECTION_NAME}/{name}/{VOLVIEW_CONFIG_NAME}") + ultrasound_config = CONFIGS_DIR / "ultrasound" / VOLVIEW_CONFIG_NAME + upload_config(gc, developer_roots["ultrasound"], ultrasound_config) + log(f" {DEVELOPER_COLLECTION_NAME}/ultrasound/{VOLVIEW_CONFIG_NAME}") + finally: + set_setting(gc, "large_image.auto_set", previous_auto_set) + log(f"Restored large_image.auto_set to {previous_auto_set!r}") + + log("\nSeeded collections:") + for name, collection in collections.items(): + log(f" {name}: {GIRDER_URL}/#collection/{collection['_id']}") + + +def cmd_seed_small(args) -> None: + """Upload the small-tier slices straight into a folder (no MinIO/S3). + + The e2e compat provisioning calls this against its own run folder: a couple + of multi-file series (patient-01 study-01 CT+PET for layering, patient-02 + study-01 CT for filtering) at --slices per series and flat item names. + girder_volview populates meta.dicom.* when each file is saved; this command + adds the study-level modality list derived from the selected series. + """ + manifest = read_manifest() + picks = small_tier_picks(manifest) + + missing = [ + p["SeriesInstanceUID"] + for p in picks + if not list(series_dir(p["SeriesInstanceUID"]).glob("*.dcm")) + ] + if missing: + die( + f"{len(missing)} small-tier series not downloaded. " + "Run `seed.py fetch --small` first." + ) + + gc = girder_client() + existing = {item["name"]: item["_id"] for item in gc.listItem(args.folder_id)} + + study_modalities: dict[str, set] = {} + for p in picks: + study_modalities.setdefault(p["StudyInstanceUID"], set()).add(p["Modality"]) + + # Plain uploads fire model.file.save, so with large_image.auto_set on these + # slices become bioformats tile sources -- which then hang the grouped item + # list at view time (and trip the O(n^2) DICOM adjacency scan). The devkit's + # `seed` disables auto_set for the same reason; do the same here. + previous_auto_set = get_setting(gc, "large_image.auto_set") + set_setting(gc, "large_image.auto_set", False) + + uploaded, skipped = 0, 0 + try: + for p in picks: + uid = p["SeriesInstanceUID"] + files = sorted_slices(list(series_dir(uid).glob("*.dcm"))) + slot = f"{p['patient_slot']}-{p['study_slot']}-{p['modality_slot']}" + modalities = sorted(study_modalities[p["StudyInstanceUID"]]) + for i, path in enumerate(subsample(files, args.slices)): + name = f"{slot}-{i:04d}.dcm" + if name in existing: + skipped += 1 + continue + file_doc = gc.uploadFileToFolder( + args.folder_id, + str(path), + filename=name, + mimeType="application/dicom", + ) + add_derived_dicom_metadata( + gc, + file_doc["itemId"], + {"ModalitiesInStudy": modalities}, + ) + uploaded += 1 + log(f" {slot}: {min(len(files), args.slices)} slices") + finally: + set_setting(gc, "large_image.auto_set", previous_auto_set) + + log( + f"Seeded small tier into folder {args.folder_id} " + f"({uploaded} uploaded, {skipped} existing)." + ) + + +def collect_layout_views(layouts: dict) -> set: + """Every bare view name mentioned across a layouts block.""" + + def views(node) -> set: + if isinstance(node, str): + return {node} + if isinstance(node, list): + return set().union(set(), *(views(n) for n in node)) + if isinstance(node, dict): + # Recurse only into `items`; `direction` holds row/column, not views. + return views(node.get("items", [])) + return set() + + return views(list(layouts.values())) + + +def find_collection(gc, name: str) -> dict | None: + return next((c for c in gc.listCollection() if c["name"] == name), None) + + +def child_folders(gc, parent_id: str) -> dict[str, dict]: + return { + folder["name"]: folder + for folder in gc.listFolder(parent_id, parentFolderType="folder") + } + + +def imaging_tree_signature(gc, root_id: str) -> set[tuple[str, ...]]: + """Return relative folder and non-config item paths below a folder.""" + paths: set[tuple[str, ...]] = set() + + def walk(folder_id: str, prefix: tuple[str, ...]) -> None: + for item in gc.listItem(folder_id): + if not item["name"].startswith("."): + paths.add(prefix + (item["name"],)) + for name, folder in child_folders(gc, folder_id).items(): + paths.add(prefix + (name,)) + walk(folder["_id"], prefix + (name,)) + + walk(root_id, ()) + return paths + + +def cmd_verify(args) -> None: + import requests + import yaml + + failures = [] + + def check(name: str, ok: bool, detail: str = "") -> None: + suffix = f" -- {detail}" if detail else "" + log(f" {'PASS' if ok else 'FAIL'} {name}{suffix}") + if not ok: + failures.append(name) + + log("Preflight") + check("girder reachable", stack_up(), API_ROOT) + check("minio reachable", minio_up(), MINIO_HOST_URL) + if failures: + die("Preflight failed.") + + gc = girder_client() + collections = {name: find_collection(gc, name) for name in COLLECTION_NAMES} + for name, collection in collections.items(): + check(f"collection {name!r} exists", collection is not None) + if any(collection is None for collection in collections.values()): + die("Required collections are missing. Run `seed.py seed`.") + + trial_roots = {} + ultrasound_roots = {} + sample_ct_folder = None + expected_patients = {f"patient-{index:02d}" for index in range(1, N_PATIENTS + 1)} + log("\nTrial hierarchies") + for collection_name in TRIAL_COLLECTION_NAMES: + collection = collections[collection_name] + roots = { + folder["name"]: folder + for folder in gc.listFolder(collection["_id"], "collection") + } + check(f"{collection_name}: patients root", "patients" in roots) + check(f"{collection_name}: ultrasound root", "ultrasound" in roots) + if "ultrasound" in roots: + ultrasound_roots[collection_name] = roots["ultrasound"] + if "patients" not in roots: + continue + patients_root = roots["patients"] + trial_roots[collection_name] = patients_root + patients = child_folders(gc, patients_root["_id"]) + check( + f"{collection_name}: {N_PATIENTS} patients", + set(patients) == expected_patients, + f"got {sorted(patients)}", + ) + for patient_name in sorted(expected_patients & patients.keys()): + studies = child_folders(gc, patients[patient_name]["_id"]) + check( + f"{collection_name}/{patient_name}: {N_STUDIES} studies", + len(studies) == N_STUDIES, + f"got {len(studies)}", + ) + for study_name, study in studies.items(): + series = child_folders(gc, study["_id"]) + check( + f"{collection_name}/{patient_name}/{study_name}: CT+PET", + {"CT", "PET"} <= set(series), + f"got {sorted(series)}", + ) + if sample_ct_folder is None and "CT" in series: + sample_ct_folder = series["CT"] + + if len(trial_roots) == len(TRIAL_COLLECTION_NAMES): + signatures = { + name: imaging_tree_signature(gc, root["_id"]) + for name, root in trial_roots.items() + } + check( + "trial collections mirror one another", + signatures[UNFILTERED_COLLECTION_NAME] + == signatures[FILTERED_COLLECTION_NAME], + f"{len(signatures[UNFILTERED_COLLECTION_NAME])} vs " + f"{len(signatures[FILTERED_COLLECTION_NAME])} paths", + ) + if len(ultrasound_roots) == len(TRIAL_COLLECTION_NAMES): + signatures = { + name: imaging_tree_signature(gc, root["_id"]) + for name, root in ultrasound_roots.items() + } + check( + "ultrasound folders mirror one another", + signatures[UNFILTERED_COLLECTION_NAME] + == signatures[FILTERED_COLLECTION_NAME], + ) + + log("\nTrial configuration") + for collection_name, root in trial_roots.items(): + names = {item["name"] for item in gc.listItem(root["_id"])} + should_filter = collection_name == FILTERED_COLLECTION_NAME + expected_state = "present" if should_filter else "absent" + check( + f"{collection_name}: large-image filter {expected_state}", + (".large_image_config.yaml" in names) == should_filter, + ) + check( + f"{collection_name}: VolView config present", + VOLVIEW_CONFIG_NAME in names, + ) + ultrasound = ultrasound_roots.get(collection_name) + if ultrasound: + ultrasound_names = {item["name"] for item in gc.listItem(ultrasound["_id"])} + check( + f"{collection_name}/ultrasound: large-image filter {expected_state}", + (".large_image_config.yaml" in ultrasound_names) == should_filter, + ) + check( + f"{collection_name}/ultrasound: VolView config present", + VOLVIEW_CONFIG_NAME in ultrasound_names, + ) + + if sample_ct_folder is not None: + items = list(gc.listItem(sample_ct_folder["_id"])) + check("CT series has items", bool(items), f"{len(items)} items") + if items: + meta = gc.getItem(items[0]["_id"]).get("meta", {}).get("dicom", {}) + for tag in ("PatientID", "StudyInstanceUID", "SeriesInstanceUID"): + check(f"meta.dicom.{tag}", bool(meta.get(tag)), str(meta.get(tag))[:40]) + + log("\nDeveloper examples") + developer = collections[DEVELOPER_COLLECTION_NAME] + developer_roots = { + folder["name"]: folder + for folder in gc.listFolder(developer["_id"], "collection") + } + check( + "developer sibling folders", + set(developer_roots) == {"prostate", "fetus", "ultrasound"}, + f"got {sorted(developer_roots)}", + ) + prostate = developer_roots.get("prostate") + fetus = developer_roots.get("fetus") + developer_ultrasound = developer_roots.get("ultrasound") + if developer_ultrasound: + names = {item["name"] for item in gc.listItem(developer_ultrasound["_id"])} + check( + "Developer/ultrasound: large-image filter absent", + ".large_image_config.yaml" not in names, + ) + check( + "Developer/ultrasound: VolView config present", + VOLVIEW_CONFIG_NAME in names, + ) + if prostate: + prostate_items = {item["name"]: item for item in gc.listItem(prostate["_id"])} + prostate_folders = child_folders(gc, prostate["_id"]) + check("prostate DICOM folder", "dicom" in prostate_folders) + check( + "prostate segmentation", + "5.seg.total-segmentator.nrrd" in prostate_items, + ) + if "dicom" in prostate_folders: + dicom_items = list(gc.listItem(prostate_folders["dicom"]["_id"])) + check( + "prostate has real DICOM", + bool(dicom_items), + f"{len(dicom_items)} slices", + ) + seg_item = prostate_items.get("5.seg.total-segmentator.nrrd") + if seg_item: + checked = gc.get( + f"folder/{prostate['_id']}/volview", + parameters={ + "folders": prostate_folders["dicom"]["_id"], + "items": seg_item["_id"], + }, + ) + checked_names = { + entry["name"] for entry in checked.get("resources", []) + } + check( + "prostate Open Checked manifest", + "5.seg.total-segmentator.nrrd" in checked_names + and any(name.endswith(".dcm") for name in checked_names), + f"got {len(checked_names)} resources", + ) + if fetus: + fetus_items = {item["name"]: item for item in gc.listItem(fetus["_id"])} + check("fetus image", "fetus.mha" in fetus_items) + check("fetus segmentation", "fetus.seg.nrrd" in fetus_items) + selected = [ + fetus_items[name]["_id"] + for name in ("fetus.mha", "fetus.seg.nrrd") + if name in fetus_items + ] + if len(selected) == 2: + checked = gc.get( + f"folder/{fetus['_id']}/volview", + parameters={"items": ",".join(selected)}, + ) + checked_names = {entry["name"] for entry in checked.get("resources", [])} + check( + "fetus Open Checked manifest", + {"fetus.mha", "fetus.seg.nrrd"} <= checked_names, + f"got {sorted(checked_names)}", + ) + + log("\nServed VolView config") + ultrasound = ultrasound_roots.get(UNFILTERED_COLLECTION_NAME) + config_targets = [] + if FILTERED_COLLECTION_NAME in trial_roots: + config_targets.append( + ( + "patients", + trial_roots[FILTERED_COLLECTION_NAME]["_id"], + CONFIGS_DIR / "trial", + ) + ) + if prostate: + config_targets.append(("prostate", prostate["_id"], CONFIGS_DIR / "developer")) + if ultrasound: + config_targets.append( + ("ultrasound", ultrasound["_id"], CONFIGS_DIR / "ultrasound") + ) + for name, folder_id, config_dir in config_targets: + local = yaml.safe_load((config_dir / VOLVIEW_CONFIG_NAME).read_text()) + bad_types = set(local.get("disabledViewTypes", [])) - VALID_DISABLED_VIEW_TYPES + check(f"{name}: disabledViewTypes valid", not bad_types, f"invalid {bad_types}") + bad_views = collect_layout_views(local.get("layouts", {})) - VALID_LAYOUT_VIEWS + check(f"{name}: layout views valid", not bad_views, f"invalid {bad_views}") + served = gc.get(f"folder/{folder_id}/volview_config/{VOLVIEW_CONFIG_NAME}") + check( + f"{name}: disabledViewTypes applied", + served.get("disabledViewTypes") == local.get("disabledViewTypes"), + f"served {served.get('disabledViewTypes')}", + ) + authored = set(local.get("layouts", {})) - {"__all__"} + check( + f"{name}: layouts applied", + authored <= set(served.get("layouts", {})), + f"served {sorted(served.get('layouts', {}))}", + ) + active = next(iter(served.get("layouts", {})), None) + check( + f"{name}: opens in {EXPECTED_ACTIVE_LAYOUT[name]!r}", + active == EXPECTED_ACTIVE_LAYOUT[name], + f"would open {active!r}", + ) + + log("\nVolView manifest") + token = gc.token + headers = {"Girder-Token": token} + manifest = ( + requests.get( + f"{API_ROOT}/folder/{sample_ct_folder['_id']}/volview", + headers=headers, + timeout=30, + ) + if sample_ct_folder + else None + ) + check( + "folder/:id/volview responds", + manifest is not None and manifest.status_code == 200, + str(manifest.status_code if manifest else "no CT folder"), + ) + if manifest is not None and manifest.status_code == 200: + resources = manifest.json().get("resources", []) + check("manifest has resources", bool(resources), f"{len(resources)} entries") + proxiable = [r for r in resources if "/proxiable/" in r.get("url", "")] + check("urls are proxiable", bool(proxiable), f"{len(proxiable)} proxiable") + if proxiable: + url = proxiable[0]["url"] + if url.startswith("/"): + url = GIRDER_URL.rstrip("/") + url + resp = requests.get( + url, headers=headers, allow_redirects=False, stream=True, timeout=30 + ) + # A 303 here means Girder handed the browser a presigned minio:9000 + # URL, which the host cannot resolve. Streaming (200) is what we want. + check( + "file streams through girder (not a redirect)", + resp.status_code == 200, + f"status {resp.status_code}", + ) + + if ultrasound: + log("\nUltrasound clips") + all_ultrasound_roots = { + **ultrasound_roots, + **( + {DEVELOPER_COLLECTION_NAME: developer_ultrasound} + if developer_ultrasound + else {} + ), + } + for collection_name, root in all_ultrasound_roots.items(): + clips = [ + item + for item in gc.listItem(root["_id"]) + if item["name"].endswith(".dcm") + ] + check( + f"{collection_name}: {N_CLIPS} clips", + len(clips) == N_CLIPS, + f"got {len(clips)}", + ) + for clip in clips: + meta = gc.getItem(clip["_id"]).get("meta", {}) + frames = meta.get("dicom", {}).get("NumberOfFrames") + check( + f"{collection_name}/{clip['name']} is cine", + bool(frames) and int(frames) > 1, + f"frames={frames}", + ) + + if failures: + die(f"{len(failures)} checks failed: {', '.join(failures)}") + log("\nAll checks passed.") + + +def cmd_reset(args) -> None: + gc = girder_client() + + for collection_name in COLLECTION_NAMES: + collection = find_collection(gc, collection_name) + if collection: + log(f"Deleting collection {collection_name!r}") + gc.delete(f"collection/{collection['_id']}") + else: + log(f"No collection {collection_name!r} to delete") + + if getattr(args, "bucket", False): + log(f"Emptying bucket {BUCKET}") + s3 = s3_client() + paginator = s3.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=BUCKET): + keys = [{"Key": o["Key"]} for o in page.get("Contents", [])] + if keys: + s3.delete_objects(Bucket=BUCKET, Delete={"Objects": keys}) + STAGED_PATH.unlink(missing_ok=True) + + log("Reset complete.") + + +def cmd_reseed(args) -> None: + """Replace the managed Girder collections using the staged MinIO objects.""" + cmd_reset(argparse.Namespace(bucket=False)) + cmd_seed(args) + + +def main() -> None: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + sub = parser.add_subparsers(dest="command", required=True) + + sub.add_parser("select", help="Query IDC and pin series into manifest/series.json") + + fetch = sub.add_parser("fetch", help="Download pinned series and verify them") + fetch.add_argument("--force", action="store_true", help="Re-download everything") + fetch.add_argument( + "--small", + action="store_true", + help="Only the small-tier series the e2e compat suite uploads", + ) + + stage = sub.add_parser("stage", help="Lay out the bucket and push to MinIO") + stage.add_argument( + "--max-slices", + type=int, + default=40, + help="Slices per CT/PET series (S3 import makes one item per slice)", + ) + stage.add_argument( + "--force", action="store_true", help="Re-upload existing objects" + ) + + sub.add_parser( + "seed", help="Create assetstore, import, set metadata, upload configs" + ) + sub.add_parser( + "reseed", help="Delete and recreate all three collections from staged data" + ) + + seed_small = sub.add_parser( + "seed-small", + help="Plain-upload the small-tier DICOM slices into a folder (no MinIO)", + ) + seed_small.add_argument( + "--folder-id", required=True, help="Girder folder to upload into" + ) + seed_small.add_argument( + "--slices", type=int, default=3, help="Slices per series (uniform stride)" + ) + + sub.add_parser("verify", help="Assert the seeded hierarchy actually works") + + reset = sub.add_parser("reset", help="Delete the seeded Girder collections") + reset.add_argument( + "--bucket", action="store_true", help="Also empty the MinIO bucket" + ) + + args = parser.parse_args() + { + "select": cmd_select, + "fetch": cmd_fetch, + "stage": cmd_stage, + "seed": cmd_seed, + "reseed": cmd_reseed, + "seed-small": cmd_seed_small, + "verify": cmd_verify, + "reset": cmd_reset, + }[args.command](args) + + +if __name__ == "__main__": + main() diff --git a/e2e/tests/compat/capture.spec.ts b/e2e/tests/compat/capture.spec.ts new file mode 100644 index 0000000..c57f83b --- /dev/null +++ b/e2e/tests/compat/capture.spec.ts @@ -0,0 +1,246 @@ +import { test, expect, APIRequestContext, Page } from '@playwright/test'; +import { plantCookie, listSessionItems } from '../../helpers/girder'; +import { + readCompatState, + appendGesture, + CompatState, + GestureId, + LaunchDescriptor, + RulerRecord, + requireFixture, +} from '../../helpers/compat-state'; +import { waitForVolViewReady, remoteSave, shot } from '../../helpers/volview'; +import { isSessionManifest, resourceNames } from '../../helpers/manifest'; +import { + gotoFolder, + checkRowByItemId, + checkRowByTexts, + fillFilterBox, + expectRow, + expectNoRow, + openInVolView, + openFromItemPage, + loginViaUI, + VolViewLaunch, +} from '../../helpers/girder-ui'; +import { + placeRuler, + readRulerMeasurements, + paintStrokes, + readSegmentGroupNames, + readDatasetNames, + selectPrimaryVolume, + addLayer, +} from '../../helpers/annotations'; +import { fetchZipSummary } from '../../helpers/session-zip'; + +// CAPTURE phase — runs against the MAIN deploy. Each test drives a real girder +// UI gesture, creates content in main's client, saves, and records the session +// item + expected content into .compat-state.json for the verify phase. +// +// Only main-era affordances may be used here: main's folder save returns NO +// resumeUrl, so session items are discovered by folder-listing diff. + +const PATIENT1 = 'ACRIN-NSCLC-FDG-PET-017'; +const PATIENT2 = 'ACRIN-NSCLC-FDG-PET-022'; +const CT_DESC = 'CT IMAGES'; +const PET_DESC = 'PET NAC OSEM'; + +function requireState(): CompatState { + const state = readCompatState(); + if (!state) throw new Error('[compat] no state — did compat.setup run in capture phase?'); + return state; +} + +async function expectFresh(launch: VolViewLaunch): Promise { + const m = await launch.manifest; + expect( + isSessionManifest(m), + `capture launch must load raw images, not a session: ${resourceNames(m)}` + ).toBeFalsy(); +} + +// Save, then identify the session item the folder-scoped save minted. +async function saveAndDiffSession( + request: APIRequestContext, + token: string, + folderId: string, + popup: Page +): Promise<{ sessionItemId: string; sessionItemName: string }> { + const before = new Set((await listSessionItems(request, token, folderId)).map((i) => i._id)); + await remoteSave(popup); + const after = await listSessionItems(request, token, folderId); + const minted = after.filter((i) => !before.has(i._id)); + expect(minted.length, 'folder save should mint exactly one new session item').toBe(1); + return { sessionItemId: minted[0]._id, sessionItemName: minted[0].name }; +} + +type CapturedContent = { + datasetNames: string[]; + rulers: RulerRecord[]; + segmentGroupNames: string[]; + petLayer: boolean; +}; + +function record( + id: GestureId, + folderId: string, + launch: LaunchDescriptor, + content: CapturedContent, + zip: Awaited>, + session?: { sessionItemId: string; sessionItemName: string } +): void { + appendGesture({ + id, + folderId, + launch, + sessionItemId: session?.sessionItemId, + sessionItemName: session?.sessionItemName, + expected: { ...content, zip }, + }); +} + +test.describe('compat capture (against main deploy)', () => { + let state: CompatState; + + test.beforeEach(async ({ context, page }) => { + state = requireState(); + await plantCookie(context, state.token); + await loginViaUI(page); + }); + + test('single-item: ruler, item-scoped save', async ({ page, request }, info) => { + const fixture = requireFixture(state, 'single-item'); + const itemId = fixture.itemIds[0]; + const launch = await openFromItemPage(page, itemId); + await expectFresh(launch); + await waitForVolViewReady(launch.popup); + await shot(launch.popup, info, 'capture-single-item-loaded'); + + await placeRuler(launch.popup); + const rulers = await readRulerMeasurements(launch.popup); + expect(rulers.length).toBe(1); + const datasetNames = await readDatasetNames(launch.popup); + await shot(launch.popup, info, 'capture-single-item-content'); + + await remoteSave(launch.popup); + const zip = await fetchZipSummary(request, state.token, itemId); + expect(zip.rulerCount).toBe(1); + + record( + 'single-item', + fixture.folderId, + { via: 'item-page', itemId }, + { datasetNames, rulers, segmentGroupNames: [], petLayer: false }, + zip + ); + }); + + test('checked-nrrd: ruler + painted segment group, folder save', async ({ page, request }, info) => { + const fixture = requireFixture(state, 'checked-nrrd'); + const itemIds = fixture.itemIds; + await gotoFolder(page, fixture.folderId); + for (const id of itemIds) await checkRowByItemId(page, id); + const launch = await openInVolView(page); + await expectFresh(launch); + await waitForVolViewReady(launch.popup); + await shot(launch.popup, info, 'capture-checked-nrrd-loaded'); + + await placeRuler(launch.popup); + const rulers = await readRulerMeasurements(launch.popup); + expect(rulers.length).toBe(1); + + await paintStrokes(launch.popup); + const segmentGroupNames = await readSegmentGroupNames(launch.popup); + expect(segmentGroupNames.length).toBeGreaterThan(0); + const datasetNames = await readDatasetNames(launch.popup); + await shot(launch.popup, info, 'capture-checked-nrrd-content'); + + const session = await saveAndDiffSession(request, state.token, fixture.folderId, launch.popup); + const zip = await fetchZipSummary(request, state.token, session.sessionItemId); + expect(zip.rulerCount).toBe(1); + expect(zip.segmentGroupCount).toBeGreaterThan(0); + expect(zip.segmentGroupDataBytes, 'painted labelmap should be non-trivial').toBeGreaterThan(0); + + record( + 'checked-nrrd', + fixture.folderId, + { via: 'checked-items', itemIds }, + { datasetNames, rulers, segmentGroupNames, petLayer: false }, + zip, + session + ); + }); + + test('filtered-dicom: filter box narrows, ruler, filter-linked save', async ({ page, request }, info) => { + const fixture = requireFixture(state, 'filtered-dicom'); + await gotoFolder(page, fixture.folderId); + // Three series rows: p1 CT, p1 PET, p2 CT. + await expectRow(page, [PATIENT1, CT_DESC]); + await expectRow(page, [PATIENT1, PET_DESC]); + await expectRow(page, [PATIENT2, CT_DESC]); + await fillFilterBox(page, PATIENT2); + await expectNoRow(page, [PATIENT1]); + await expectRow(page, [PATIENT2, CT_DESC]); + await checkRowByTexts(page, [PATIENT2]); + const launch = await openInVolView(page); + await expectFresh(launch); + await waitForVolViewReady(launch.popup); + await shot(launch.popup, info, 'capture-filtered-dicom-loaded'); + + await placeRuler(launch.popup); + const rulers = await readRulerMeasurements(launch.popup); + expect(rulers.length).toBe(1); + const datasetNames = await readDatasetNames(launch.popup); + await shot(launch.popup, info, 'capture-filtered-dicom-content'); + + const session = await saveAndDiffSession(request, state.token, fixture.folderId, launch.popup); + const zip = await fetchZipSummary(request, state.token, session.sessionItemId); + expect(zip.rulerCount).toBe(1); + + record( + 'filtered-dicom', + fixture.folderId, + { via: 'checked-rows', rows: [[PATIENT2]], filterText: PATIENT2 }, + { datasetNames, rulers, segmentGroupNames: [], petLayer: false }, + zip, + session + ); + }); + + test('study-layered: CT+PET checked, PET layered over CT, ruler', async ({ page, request }, info) => { + const fixture = requireFixture(state, 'study-layered'); + const rows = [ + [PATIENT1, CT_DESC], + [PATIENT1, PET_DESC], + ]; + await gotoFolder(page, fixture.folderId); + for (const row of rows) await checkRowByTexts(page, row); + const launch = await openInVolView(page); + await expectFresh(launch); + await waitForVolViewReady(launch.popup); + await shot(launch.popup, info, 'capture-study-layered-loaded'); + + await selectPrimaryVolume(launch.popup, CT_DESC); + await addLayer(launch.popup, PET_DESC); + await placeRuler(launch.popup); + const rulers = await readRulerMeasurements(launch.popup); + expect(rulers.length).toBe(1); + const datasetNames = await readDatasetNames(launch.popup); + await shot(launch.popup, info, 'capture-study-layered-content'); + + const session = await saveAndDiffSession(request, state.token, fixture.folderId, launch.popup); + const zip = await fetchZipSummary(request, state.token, session.sessionItemId); + expect(zip.rulerCount).toBe(1); + expect(zip.hasLayers, 'the PET layer should serialize into the session').toBeTruthy(); + + record( + 'study-layered', + fixture.folderId, + { via: 'checked-rows', rows }, + { datasetNames, rulers, segmentGroupNames: [], petLayer: true }, + zip, + session + ); + }); +}); diff --git a/e2e/tests/compat/devkit.capture.spec.ts b/e2e/tests/compat/devkit.capture.spec.ts new file mode 100644 index 0000000..83ad2d0 --- /dev/null +++ b/e2e/tests/compat/devkit.capture.spec.ts @@ -0,0 +1,91 @@ +import { test, expect } from '@playwright/test'; +import { plantCookie, listSessionItems } from '../../helpers/girder'; +import { readCompatState, appendGesture, CompatState } from '../../helpers/compat-state'; +import { waitForVolViewReady, remoteSave, shot } from '../../helpers/volview'; +import { isSessionManifest } from '../../helpers/manifest'; +import { gotoFolder, drillRowNav, loginViaUI } from '../../helpers/girder-ui'; +import { + placeRuler, + readRulerMeasurements, + readDatasetNames, + selectPrimaryVolume, + addLayer, +} from '../../helpers/annotations'; +import { fetchZipSummary } from '../../helpers/session-zip'; +import { apiUrl } from '../../helpers/config'; + +// Tier 2 (optional): the fully-seeded devkit collection — real patient→study +// drill-down through the .large_image_config.yaml hierarchy, whole-study +// CT+PET launch. Skipped unless `seed.py seed` has run against this stack. + +const PATIENT1 = 'ACRIN-NSCLC-FDG-PET-017'; +const CT_DESC = 'CT IMAGES'; +const PET_DESC = 'PET NAC OSEM'; + +test.describe('compat capture: devkit study drill-down', () => { + let state: CompatState; + + test.beforeEach(async ({ context, page }) => { + const s = readCompatState(); + if (!s) throw new Error('[compat] no state — did compat.setup run in capture phase?'); + state = s; + await plantCookie(context, state.token); + await loginViaUI(page); + }); + + test('devkit-study: patient → study row opens whole study; layer + ruler; save', async ({ + page, + request, + }, info) => { + test.skip(!state.devkitTrialFolderId, 'VolView Devkit collection not seeded (optional tier)'); + const folderId = state.devkitTrialFolderId!; + const rowTexts = [PATIENT1, PATIENT1]; // patient row, then its first study row + + // The devkit collection is shared and persistent — session zips left by + // earlier runs make the drill-down resume instead of loading fresh. Clear + // them so capture is idempotent. + const stale = await listSessionItems(request, state.token, folderId); + for (const item of stale) { + await request.delete(apiUrl(`/item/${item._id}`), { + headers: { 'Girder-Token': state.token }, + }); + } + + await gotoFolder(page, folderId); + const launch = await drillRowNav(page, rowTexts); + const m = await launch.manifest; + expect(isSessionManifest(m), 'devkit study launch must be fresh').toBeFalsy(); + await waitForVolViewReady(launch.popup); + await shot(launch.popup, info, 'capture-devkit-study-loaded'); + + await selectPrimaryVolume(launch.popup, CT_DESC); + await addLayer(launch.popup, PET_DESC); + await placeRuler(launch.popup); + const rulers = await readRulerMeasurements(launch.popup); + expect(rulers.length).toBe(1); + const datasetNames = await readDatasetNames(launch.popup); + await shot(launch.popup, info, 'capture-devkit-study-content'); + + const before = new Set( + (await listSessionItems(request, state.token, folderId)).map((i) => i._id) + ); + await remoteSave(launch.popup); + const minted = (await listSessionItems(request, state.token, folderId)).filter( + (i) => !before.has(i._id) + ); + expect(minted.length, 'devkit study save should mint one session item').toBe(1); + + const zip = await fetchZipSummary(request, state.token, minted[0]._id); + expect(zip.rulerCount).toBe(1); + expect(zip.hasLayers).toBeTruthy(); + + appendGesture({ + id: 'devkit-study', + folderId, + launch: { via: 'row-nav', rowTexts }, + sessionItemId: minted[0]._id, + sessionItemName: minted[0].name, + expected: { datasetNames, rulers, segmentGroupNames: [], petLayer: true, zip }, + }); + }); +}); diff --git a/e2e/tests/compat/devkit.verify.spec.ts b/e2e/tests/compat/devkit.verify.spec.ts new file mode 100644 index 0000000..4563c0a --- /dev/null +++ b/e2e/tests/compat/devkit.verify.spec.ts @@ -0,0 +1,74 @@ +import { test, expect } from '@playwright/test'; +import { plantCookie, listSessionItems } from '../../helpers/girder'; +import { readCompatState, CompatState } from '../../helpers/compat-state'; +import { waitForVolViewReady, remoteSave, urlsParam, shot } from '../../helpers/volview'; +import { isSessionManifest, resourceNames } from '../../helpers/manifest'; +import { gotoFolder, drillRowNav, loginViaUI } from '../../helpers/girder-ui'; +import { readRulerMeasurements, isLayered } from '../../helpers/annotations'; +import { fetchZipSummary } from '../../helpers/session-zip'; +import { apiUrl } from '../../helpers/config'; + +const PET_DESC = 'PET NAC OSEM'; + +test.describe('compat verify: devkit study drill-down', () => { + let state: CompatState; + + test.beforeEach(async ({ context, page }) => { + const s = readCompatState(); + if (!s) throw new Error('[compat] no state — run the capture phase first'); + state = s; + await plantCookie(context, state.token); + await loginViaUI(page); + }); + + test('devkit-study: replaying the drill-down resumes the session with its layer', async ({ + page, + request, + }, info) => { + const gesture = state.gestures.find((g) => g.id === 'devkit-study'); + test.skip(!gesture, 'devkit tier was not captured (optional)'); + const { rowTexts } = gesture!.launch as { rowTexts: string[] }; + const folderId = gesture!.folderId; + + await gotoFolder(page, folderId); + const launch = await drillRowNav(page, rowTexts); + const m = await launch.manifest; + expect( + isSessionManifest(m), + `study drill-down must resume the main-era session: ${resourceNames(m)}` + ).toBeTruthy(); + expect(resourceNames(m)).toContain(gesture!.sessionItemName); + await waitForVolViewReady(launch.popup); + await shot(launch.popup, info, 'verify-devkit-study-restored'); + + const rulers = await readRulerMeasurements(launch.popup); + expect(rulers.map((r) => r.lengthText).sort()).toEqual( + gesture!.expected.rulers.map((r) => r.lengthText).sort() + ); + expect(await isLayered(launch.popup, PET_DESC), 'PET layer lost in restore').toBeTruthy(); + + // Branch re-save round-trip. + const before = new Set( + (await listSessionItems(request, state.token, folderId)).map((i) => i._id) + ); + const resumeUrl = await remoteSave(launch.popup); + expect(resumeUrl, 'branch save must return a resumeUrl').toBeTruthy(); + expect(urlsParam(launch.popup)).toBe(resumeUrl); + const minted = (await listSessionItems(request, state.token, folderId)).filter( + (i) => !before.has(i._id) + ); + expect(minted.length).toBe(1); + const zip = await fetchZipSummary(request, state.token, minted[0]._id); + expect(zip.rulerCount).toBe(gesture!.expected.zip.rulerCount); + expect(zip.hasLayers).toBeTruthy(); + + // The devkit collection is shared, not run-provisioned: remove the session + // items this run minted so re-runs start clean. + for (const itemId of [gesture!.sessionItemId, minted[0]._id]) { + if (!itemId) continue; + await request.delete(apiUrl(`/item/${itemId}`), { + headers: { 'Girder-Token': state.token }, + }); + } + }); +}); diff --git a/e2e/tests/compat/verify.spec.ts b/e2e/tests/compat/verify.spec.ts new file mode 100644 index 0000000..cf8a99f --- /dev/null +++ b/e2e/tests/compat/verify.spec.ts @@ -0,0 +1,216 @@ +import { test, expect, APIRequestContext, Page } from '@playwright/test'; +import { plantCookie, listSessionItems } from '../../helpers/girder'; +import { + readCompatState, + CompatState, + CapturedGesture, + ZipSummary, +} from '../../helpers/compat-state'; +import { waitForVolViewReady, remoteSave, urlsParam, shot } from '../../helpers/volview'; +import { + isSessionManifest, + resourceNames, + reloadCapturingManifest, +} from '../../helpers/manifest'; +import { + gotoFolder, + checkRowByItemId, + checkRowByTexts, + fillFilterBox, + openInVolView, + openFromItemPage, + loginViaUI, + VolViewLaunch, +} from '../../helpers/girder-ui'; +import { + readRulerMeasurements, + readSegmentGroupNames, + isLayered, +} from '../../helpers/annotations'; +import { fetchZipSummary } from '../../helpers/session-zip'; + +// VERIFY phase — runs against THIS worktree's deploy, after the redeploy. +// Each captured main-era session must: resolve through the branch's manifest +// logic when the gesture is replayed, restore its content faithfully, and +// round-trip through a branch re-save + F5. + +const PET_DESC = 'PET NAC OSEM'; + +function requireGesture(state: CompatState, id: CapturedGesture['id']): CapturedGesture { + const gesture = state.gestures.find((g) => g.id === id); + if (!gesture) throw new Error(`[compat] capture did not record gesture '${id}'`); + return gesture; +} + +const sortedLengths = (rulers: Array<{ lengthText: string }>) => + rulers.map((r) => r.lengthText).sort(); + +async function assertContentRestored(popup: Page, gesture: CapturedGesture): Promise { + const rulers = await readRulerMeasurements(popup); + expect( + sortedLengths(rulers), + 'restored ruler measurements must match the capture exactly (world coords live in the zip)' + ).toEqual(sortedLengths(gesture.expected.rulers)); + + if (gesture.expected.segmentGroupNames.length) { + const groups = await readSegmentGroupNames(popup); + for (const name of gesture.expected.segmentGroupNames) { + expect(groups, `segment group "${name}" lost in restore`).toContain(name); + } + } + + if (gesture.expected.petLayer) { + expect(await isLayered(popup, PET_DESC), 'PET layer lost in restore').toBeTruthy(); + } +} + +// The main-era zip's content must round-trip through the BRANCH serializer: +// re-save and compare semantic summaries (schema migrations are fine; content +// loss is not). +function expectZipRoundTrip(fresh: ZipSummary, gesture: CapturedGesture): void { + const captured = gesture.expected.zip; + expect(fresh.rulerCount, 're-saved zip lost rulers').toBe(captured.rulerCount); + expect(fresh.segmentGroupCount, 're-saved zip lost segment groups').toBeGreaterThanOrEqual( + captured.segmentGroupCount + ); + if (captured.segmentGroupDataBytes > 0) { + expect(fresh.segmentGroupDataBytes, 're-saved labelmap is empty').toBeGreaterThan(0); + } + if (gesture.expected.petLayer) { + expect(fresh.hasLayers, 're-saved zip lost the layer').toBeTruthy(); + } +} + +async function expectResumedSession(launch: VolViewLaunch, sessionItemName?: string): Promise { + const m = await launch.manifest; + expect( + isSessionManifest(m), + `branch must resume the main-era session: ${resourceNames(m)}` + ).toBeTruthy(); + if (sessionItemName) { + expect(resourceNames(m), 'manifest names a different session').toContain(sessionItemName); + } +} + +// Branch re-save from a resumed main session, then F5 must reload the NEW save. +async function resaveAndReload( + request: APIRequestContext, + state: CompatState, + gesture: CapturedGesture, + popup: Page +): Promise { + const folderScoped = gesture.launch.via !== 'item-page'; + const before = new Set( + (await listSessionItems(request, state.token, gesture.folderId)).map((i) => i._id) + ); + + const resumeUrl = await remoteSave(popup); + expect(resumeUrl, 'branch save must return a resumeUrl').toBeTruthy(); + expect(urlsParam(popup)).toBe(resumeUrl); + + let zipItemId: string; + if (folderScoped) { + const after = await listSessionItems(request, state.token, gesture.folderId); + const minted = after.filter((i) => !before.has(i._id)); + expect(minted.length, 'branch folder save should mint a new session item').toBe(1); + zipItemId = minted[0]._id; + } else { + zipItemId = (gesture.launch as { itemId: string }).itemId; + } + expectZipRoundTrip(await fetchZipSummary(request, state.token, zipItemId), gesture); + + const m = await reloadCapturingManifest(popup); + expect(urlsParam(popup), 'F5 after the branch save must stay on its resumeUrl').toBe(resumeUrl); + expect(isSessionManifest(m), 'F5 must resume the branch save').toBeTruthy(); + await assertContentRestored(popup, gesture); +} + +test.describe('compat verify (against branch deploy)', () => { + let state: CompatState; + + test.beforeEach(async ({ context, page }) => { + const s = readCompatState(); + if (!s) throw new Error('[compat] no state — run the capture phase first'); + state = s; + await plantCookie(context, state.token); + await loginViaUI(page); + }); + + test('single-item: item manifest serves the main-era session', async ({ page, request }, info) => { + const gesture = requireGesture(state, 'single-item'); + const { itemId } = gesture.launch as { itemId: string }; + + const launch = await openFromItemPage(page, itemId); + await expectResumedSession(launch); + await waitForVolViewReady(launch.popup); + await shot(launch.popup, info, 'verify-single-item-restored'); + await assertContentRestored(launch.popup, gesture); + + await resaveAndReload(request, state, gesture, launch.popup); + }); + + test('checked-nrrd: bare-folder open resumes, session-row open targets it', async ({ + page, + request, + }, info) => { + const gesture = requireGesture(state, 'checked-nrrd'); + + // Bare folder-open (nothing checked) must resume the newest session — the + // one main saved. + await gotoFolder(page, gesture.folderId); + const launch = await openInVolView(page); + await expectResumedSession(launch, gesture.sessionItemName); + await waitForVolViewReady(launch.popup); + await shot(launch.popup, info, 'verify-checked-nrrd-restored'); + await assertContentRestored(launch.popup, gesture); + + // Checking the main-era session item (plus a raw image) in the girder UI + // must open exactly that session. + await gotoFolder(page, gesture.folderId); + await checkRowByItemId(page, gesture.sessionItemId!); + await checkRowByItemId(page, (gesture.launch as { itemIds: string[] }).itemIds[0]); + const viaRow = await openInVolView(page); + await waitForVolViewReady(viaRow.popup); + expect(urlsParam(viaRow.popup)).toContain(`items=${gesture.sessionItemId}`); + await viaRow.popup.close(); + + await resaveAndReload(request, state, gesture, launch.popup); + }); + + test('filtered-dicom: replaying the filter gesture resumes the matching session', async ({ + page, + request, + }, info) => { + const gesture = requireGesture(state, 'filtered-dicom'); + const launch0 = gesture.launch as { rows: string[][]; filterText?: string }; + + await gotoFolder(page, gesture.folderId); + if (launch0.filterText) await fillFilterBox(page, launch0.filterText); + for (const row of launch0.rows) await checkRowByTexts(page, row); + const launch = await openInVolView(page); + await expectResumedSession(launch, gesture.sessionItemName); + await waitForVolViewReady(launch.popup); + await shot(launch.popup, info, 'verify-filtered-dicom-restored'); + await assertContentRestored(launch.popup, gesture); + + await resaveAndReload(request, state, gesture, launch.popup); + }); + + test('study-layered: replaying the CT+PET selection resumes with the layer', async ({ + page, + request, + }, info) => { + const gesture = requireGesture(state, 'study-layered'); + const launch0 = gesture.launch as { rows: string[][] }; + + await gotoFolder(page, gesture.folderId); + for (const row of launch0.rows) await checkRowByTexts(page, row); + const launch = await openInVolView(page); + await expectResumedSession(launch, gesture.sessionItemName); + await waitForVolViewReady(launch.popup); + await shot(launch.popup, info, 'verify-study-layered-restored'); + await assertContentRestored(launch.popup, gesture); + + await resaveAndReload(request, state, gesture, launch.popup); + }); +}); diff --git a/e2e/tests/run-and-apply.spec.ts b/e2e/tests/run-and-apply.spec.ts new file mode 100644 index 0000000..b703586 --- /dev/null +++ b/e2e/tests/run-and-apply.spec.ts @@ -0,0 +1,179 @@ +import { test, expect, Page } from '@playwright/test'; +import { setupFixture, requireHarnessState, Girder } from '../helpers/girder'; +import { requireFixture } from '../helpers/compat-state'; +import { gotoFolder, checkRowByItemId, openInVolView } from '../helpers/girder-ui'; +import { submitOtsu } from '../helpers/jobs'; +import { + waitForVolViewReady, + openModuleTab, + loadJobResults, + selectTask, + waitForInputBound, + submitTaskFromForm, + waitForJobComplete, + shot, +} from '../helpers/volview'; +import { paintStrokes, readDatasetNames, readSegmentGroupNames } from '../helpers/annotations'; + +// The jobs/processing plane in the browser. Setup submits an Otsu job over REST +// (folder+user scoped to the same admin the tab runs as) and polls it to +// success, so the launched tab meets a job that finished before it existed and +// must reach it through the come-back path. The launch carries config=, which +// is what makes the Jobs tab appear. + +async function launchChecked(driver: Page, g: Girder): Promise { + await gotoFolder(driver, g.folderId); + await checkRowByItemId(driver, g.itemId); + const launch = await openInVolView(driver); + await waitForVolViewReady(launch.popup); + return launch.popup; +} + +test.describe('jobs come-back path (Load results)', () => { + // Submit the Otsu job once (REST) for the whole suite; the launched tab just + // loads its result. + test.beforeAll(async ({ request }) => { + const state0 = requireHarnessState(); + const fixture = requireFixture(state0, 'jobs-comeback'); + const { jobId, state } = await submitOtsu( + request, + state0.token, + fixture.folderId, + fixture.itemIds[0] + ); + expect(state, `Otsu job ${jobId} did not succeed (state=${state})`).toBe('success'); + // eslint-disable-next-line no-console + console.log( + `[e2e] Otsu job ${jobId} succeeded — folder ${fixture.folderId} ready for come-back test` + ); + }); + + test('Load results applies the labelmap as a segment group on the original image', async ({ + page, + context, + }, info) => { + const g = await setupFixture(context, 'jobs-comeback'); + const view = await launchChecked(page, g); + + // The come-back job must NOT auto-apply: before the explicit load there is + // no Otsu segment group and the Load action is still available. + await openModuleTab(view, 'Annotations'); + await expect( + view.locator('.segment-group-list').getByText(/Otsu/), + 'a history job auto-applied without "Load"' + ).toHaveCount(0); + + await loadJobResults(view); + await shot(view, info, 'jobs-tab-results'); + // The button is consumed: loading = applying, exactly once. + await expect( + view.locator('.jobs-module').getByRole('button', { name: 'Load', exact: true }) + ).toHaveCount(0); + + // Intent-honoring apply: the labelmap became an "." segment + // group on the reconstructed parent image (no manual verb choice). + await openModuleTab(view, 'Annotations'); + await expect( + view.locator('.segment-group-list').getByText(/Otsu/).first(), + 'no Otsu segment group in the segment-group list' + ).toBeVisible({ timeout: 30_000 }); + await shot(view, info, 'come-back-apply'); + }); +}); + +// Drives the full visible UI path — task picker, form, provenance binding, +// Submit, poll, result stream, live auto-apply — under the product's cookie +// auth (the girder launcher's popup shares the session cookie; this girder +// does not honor Authorization: Bearer). The come-back suite above covers the +// explicit "Load" path only. +test.describe('live submission + auto-apply (the submission gate)', () => { + test('submits from the UI and live-auto-applies the result', async ({ + page, + context, + }, info) => { + const g = await setupFixture(context, 'jobs-live'); + const view = await launchChecked(page, g); + + // Result-byte reads go through proxiable file URLs; count them to prove the + // result stream actually flowed. + const fileReads: string[] = []; + view.on('request', (r) => { + if (/\/file\/[^/]+\/proxiable\//.test(r.url())) fileReads.push(r.url()); + }); + + // Drive the VISIBLE submission flow: task picker -> binding -> Submit. + await selectTask(view, 'Otsu'); + await waitForInputBound(view); + await submitTaskFromForm(view); + + // Poll to live completion (the store's own toast), then confirm LIVE + // auto-apply attached the result with NO manual "Load" click: + // the Otsu labelmap becomes an "." segment group. + await waitForJobComplete(view); + await openModuleTab(view, 'Annotations'); + await expect( + view.locator('.segment-group-list').getByText(/Otsu/).first(), + 'live auto-apply did not attach a segment group' + ).toBeVisible({ timeout: 30_000 }); + await shot(view, info, 'live-auto-apply'); + + expect(fileReads.length, 'no proxiable result file read observed').toBeGreaterThan(0); + }); + + test('stages a painted mask and auto-applies the filtered image result', async ({ + page, + context, + }, info) => { + const g = await setupFixture(context, 'jobs-staged'); + const view = await launchChecked(page, g); + const datasetsBefore = await readDatasetNames(view); + + await paintStrokes(view); + expect(await readSegmentGroupNames(view)).not.toEqual([]); + + await selectTask(view, 'MaskedMedianFilter'); + await expect( + view.locator('.jobs-module').getByText('Active segment group').first(), + 'the painted segment group was not bound as the filter mask' + ).toBeVisible(); + await submitTaskFromForm(view); + await waitForJobComplete(view); + + const datasetsAfter = await readDatasetNames(view); + expect( + datasetsAfter.filter((name) => !datasetsBefore.includes(name)), + 'the filtered image result was not added to the scene' + ).not.toEqual([]); + await shot(view, info, 'staged-mask-image-result'); + }); + + test('shows a failed job error and deletes the terminal job', async ({ page, context }) => { + const g = await setupFixture(context, 'jobs-failure'); + const view = await launchChecked(page, g); + + await selectTask(view, 'ThresholdSegmentation'); + await waitForInputBound(view); + const thresholds = view.locator('.jobs-module input[type="number"]'); + await expect(thresholds).toHaveCount(2); + await thresholds.nth(0).fill('200'); + await thresholds.nth(1).fill('100'); + await submitTaskFromForm(view); + + const row = view.locator('.job-row').filter({ hasText: 'Threshold Segmentation' }).first(); + await row.getByRole('button', { name: 'Details', exact: true }).click(); + await expect(row.locator('.job-subtitle'), 'the threshold job did not reach Failed').toContainText( + /^Failed\b/, + { timeout: 180_000 } + ); + await expect(row.getByRole('button', { name: 'Load', exact: true })).toHaveCount(0); + await expect(row.locator('.error-log'), 'the failed job exposes no error details').toContainText( + /Lower threshold/ + ); + + await row.getByLabel('Delete job').click(); + const dialog = view.getByText('Delete job?').locator('..'); + await expect(dialog).toContainText('This cannot be undone'); + await dialog.getByRole('button', { name: 'Delete', exact: true }).click(); + await expect(row, 'the deleted job remained in history').toHaveCount(0); + }); +}); diff --git a/e2e/tests/save-load-restore.spec.ts b/e2e/tests/save-load-restore.spec.ts new file mode 100644 index 0000000..3f3834a --- /dev/null +++ b/e2e/tests/save-load-restore.spec.ts @@ -0,0 +1,300 @@ +import { test, expect, Page } from '@playwright/test'; +import { + gotoFolder, + loginViaUI, + checkRowByItemId, + checkRowByTexts, + fillFilterBox, + uncheckAllRows, + openInVolView, + openFromItemPage, +} from '../helpers/girder-ui'; +import { + setupFixture, + countSessionItems, + firstFileId, + fetchManifest, + resourceUrls, + CONFIG, + Girder, +} from '../helpers/girder'; +import { FixtureId } from '../helpers/compat-state'; +import { + waitForVolViewReady, + urlsParam, + remoteSave, + shot, +} from '../helpers/volview'; +import { placeRuler, readRulerMeasurements } from '../helpers/annotations'; +import { + isSessionManifest, + resourceNames, + reloadCapturingManifest, +} from '../helpers/manifest'; + +// Current-to-current lifecycle coverage. Every scenario owns a provisioned +// folder, and every launch goes through the deployed Girder UI and open.js. + +const PATIENT2 = 'ACRIN-NSCLC-FDG-PET-022'; + +type Gesture = 'single-item' | 'checked' | 'filter' | 'bare-folder'; +type Launched = { view: Page; freshManifest: string; manifest: any }; + +async function launchGesture(driver: Page, g: Girder, gesture: Gesture): Promise { + let launch; + if (gesture === 'single-item') { + launch = await openFromItemPage(driver, g.itemId); + } else { + await gotoFolder(driver, g.folderId); + if (gesture === 'checked') { + await checkRowByItemId(driver, g.itemId); + } else if (gesture === 'filter') { + await fillFilterBox(driver, PATIENT2); + await checkRowByTexts(driver, [PATIENT2]); + } else { + await uncheckAllRows(driver); + } + launch = await openInVolView(driver); + } + await waitForVolViewReady(launch.popup); + return { + view: launch.popup, + freshManifest: urlsParam(launch.popup), + manifest: await launch.manifest, + }; +} + +function expectFreshRoute(g: Girder, gesture: Gesture, manifestUrl: string): void { + const route = new URL(manifestUrl, CONFIG.baseURL); + if (gesture === 'single-item') { + expect(route.pathname).toBe(`/${CONFIG.apiRoot}/item/${g.itemId}/volview`); + return; + } + expect(route.pathname).toBe(`/${CONFIG.apiRoot}/folder/${g.folderId}/volview`); + if (gesture === 'checked') { + expect(route.searchParams.get('items')).toBe(g.itemId); + expect(route.searchParams.has('folders')).toBeTruthy(); + } else if (gesture === 'filter') { + expect(route.searchParams.has('filters'), 'grouped launch emitted no filters= leg').toBeTruthy(); + expect(JSON.parse(route.searchParams.get('filters') || '[]')).not.toEqual([]); + } else { + expect(route.searchParams.has('items')).toBeTruthy(); + expect(route.searchParams.get('items')).toBe(''); + expect(route.searchParams.has('folders')).toBeTruthy(); + expect(route.searchParams.get('folders')).toBe(''); + } +} + +test.describe('save/load/restore F5 lifecycle', () => { + const cases: Array<{ gesture: Exclude; fixture: FixtureId }> = [ + { gesture: 'single-item', fixture: 'lifecycle-single' }, + { gesture: 'checked', fixture: 'lifecycle-checked' }, + { gesture: 'filter', fixture: 'lifecycle-filter' }, + ]; + + for (const { gesture, fixture } of cases) { + test(`${gesture}: fresh -> F5-stays-fresh -> save -> F5-resumes -> save-again -> F5-resumes`, async ({ + page, + context, + }, info) => { + const g = await setupFixture(context, fixture); + const launched = await launchGesture(page, g, gesture); + const view = launched.view; + const freshManifest = launched.freshManifest; + const m1 = launched.manifest; + await shot(view, info, `${gesture}-1-launch-fresh`); + expectFreshRoute(g, gesture, freshManifest); + expect( + isSessionManifest(m1), + `fresh launch must not load a session zip: ${resourceNames(m1)}` + ).toBeFalsy(); + expect(resourceNames(m1).some((name) => name !== 'config.json')).toBeTruthy(); + + const m2 = await reloadCapturingManifest(view); + await shot(view, info, `${gesture}-2-f5-stays-fresh`); + expect(urlsParam(view), 'F5-before-save must not repoint').toBe(freshManifest); + expect(isSessionManifest(m2), 'F5-before-save pulled in a session').toBeFalsy(); + + await placeRuler(view); + const savedRulers = await readRulerMeasurements(view); + expect(savedRulers).toHaveLength(1); + + const sessionsBefore = await countSessionItems(page.request, g); + const resumeUrl1 = await remoteSave(view); + await shot(view, info, `${gesture}-3-after-save`); + expect(resumeUrl1, 'save response carried no resumeUrl').toBeTruthy(); + expect(urlsParam(view), 'urls= must repoint to the save resumeUrl').toBe(resumeUrl1); + if (gesture !== 'single-item') { + expect(await countSessionItems(page.request, g)).toBeGreaterThan(sessionsBefore); + } + + const m4 = await reloadCapturingManifest(view); + await shot(view, info, `${gesture}-4-f5-resumes-save`); + expect(urlsParam(view), 'F5-after-save must stay on the resumeUrl').toBe(resumeUrl1); + expect(isSessionManifest(m4), 'F5-after-save did not load the session').toBeTruthy(); + expect(await readRulerMeasurements(view), 'F5-after-save lost the saved ruler').toEqual( + savedRulers + ); + + if (gesture === 'filter' || gesture === 'checked') { + const reopenDriver = await context.newPage(); + const reopened = await launchGesture(reopenDriver, g, gesture); + const shouldResume = gesture === 'filter'; + expect( + isSessionManifest(reopened.manifest), + shouldResume + ? `reopening filter did not resume its save: ${resourceNames(reopened.manifest)}` + : `reopening checked raw images did not start fresh: ${resourceNames(reopened.manifest)}` + ).toBe(shouldResume); + await reopened.view.close(); + await reopenDriver.close(); + } + + const resumeUrl2 = await remoteSave(view); + await shot(view, info, `${gesture}-5-after-second-save`); + expect(resumeUrl2, 'second save carried no resumeUrl').toBeTruthy(); + expect(urlsParam(view)).toBe(resumeUrl2); + await reloadCapturingManifest(view); + await shot(view, info, `${gesture}-6-f5-resumes-second-save`); + expect(urlsParam(view), 'F5 after the second save left its resumeUrl').toBe(resumeUrl2); + }); + } + + test('fresh restart via checked raw images starts clean, then F5 resumes the new save', async ({ + page, + context, + }, info) => { + const g = await setupFixture(context, 'lifecycle-restart'); + const seed = await launchGesture(page, g, 'checked'); + const olderResume = await remoteSave(seed.view); + expect(olderResume, 'seeding save carried no resumeUrl').toBeTruthy(); + await seed.view.close(); + + const restart = await launchGesture(page, g, 'checked'); + await shot(restart.view, info, 'restart-1-fresh-despite-older-save'); + expectFreshRoute(g, 'checked', restart.freshManifest); + expect( + isSessionManifest(restart.manifest), + `restart resumed the older save: ${resourceNames(restart.manifest)}` + ).toBeFalsy(); + + const newResume = await remoteSave(restart.view); + await shot(restart.view, info, 'restart-2-after-save'); + expect(newResume, 'restart save carried no resumeUrl').toBeTruthy(); + expect(newResume, 'the new save reused the older session item').not.toBe(olderResume); + + const manifest = await reloadCapturingManifest(restart.view); + await shot(restart.view, info, 'restart-3-f5-resumes-new-save'); + expect(urlsParam(restart.view)).toBe(newResume); + expect(isSessionManifest(manifest)).toBeTruthy(); + }); + + test('bare folder-open resumes the newest folder-scoped save', async ({ page, context }, info) => { + const g = await setupFixture(context, 'lifecycle-bare'); + const seed = await launchGesture(page, g, 'checked'); + const seededResume = await remoteSave(seed.view); + expect(seededResume).toBeTruthy(); + await seed.view.close(); + + const bare = await launchGesture(page, g, 'bare-folder'); + await shot(bare.view, info, 'bare-folder-resumes-newest'); + expectFreshRoute(g, 'bare-folder', bare.freshManifest); + expect( + isSessionManifest(bare.manifest), + `bare open did not resume a session: ${resourceNames(bare.manifest)}` + ).toBeTruthy(); + }); + + test('checking an older session opens that save, not the newest', async ({ page, context }, info) => { + const g = await setupFixture(context, 'lifecycle-older'); + const checked = await launchGesture(page, g, 'checked'); + await placeRuler(checked.view); + const olderRulers = await readRulerMeasurements(checked.view); + const olderResume = await remoteSave(checked.view); + const newerResume = await remoteSave(checked.view); + expect(olderResume).toBeTruthy(); + expect(newerResume).toBeTruthy(); + expect(newerResume).not.toBe(olderResume); + await checked.view.close(); + + const idOf = (resumeUrl: string) => resumeUrl.split('/item/')[1].split('/volview')[0]; + const olderId = idOf(olderResume); + const newerId = idOf(newerResume); + const olderFileId = await firstFileId(page.request, g.token, olderId); + const newerFileId = await firstFileId(page.request, g.token, newerId); + expect(olderFileId).not.toBe(newerFileId); + + // Session items are private even though the fixture folder and raw images + // are public. Authenticate the Girder client before browsing those rows. + await loginViaUI(page); + await gotoFolder(page, g.folderId); + await checkRowByItemId(page, olderId); + const launch = await openInVolView(page); + await waitForVolViewReady(launch.popup); + await shot(launch.popup, info, 'older-session-reopened'); + + const urls = urlsParam(launch.popup); + expect(urls).toContain(`items=${olderId}`); + expect(urls).not.toContain(`items=${newerId}`); + const manifest = await fetchManifest(page.request, g.token, urls); + const manifestUrls = resourceUrls(manifest).join(' '); + expect(manifestUrls, 'the older save was not loaded').toContain(`/file/${olderFileId}/`); + expect(manifestUrls, 'the newest save was substituted').not.toContain(`/file/${newerFileId}/`); + expect(await readRulerMeasurements(launch.popup), 'the selected older session lost its ruler').toEqual( + olderRulers + ); + }); + + test('the checked-image button carries the complete launch contract', async ({ page, context }) => { + const g = await setupFixture(context, 'lifecycle-url-contract'); + await gotoFolder(page, g.folderId); + await checkRowByItemId(page, g.itemId); + + const button = page.locator('.open-in-volview'); + await expect(button).toHaveText(/Open Checked in VolView/); + const href = await button.getAttribute('href'); + expect(href, 'the open button carries no href').toBeTruthy(); + + const actual = new URL(href!, CONFIG.baseURL); + expect(actual.pathname).toBe('/static/built/plugins/volview/index.html'); + expect(actual.searchParams.get('names')).toBe('[manifest.json]'); + expect(actual.searchParams.get('config')).toBe( + `/${CONFIG.apiRoot}/folder/${g.folderId}/volview_config/.volview_config.yaml` + ); + + const manifest = new URL(actual.searchParams.get('urls')!, CONFIG.baseURL); + expect(manifest.pathname).toBe(`/${CONFIG.apiRoot}/folder/${g.folderId}/volview`); + expect(manifest.searchParams.has('folders')).toBeTruthy(); + expect(manifest.searchParams.get('folders')).toBe(''); + expect(manifest.searchParams.get('items')).toBe(g.itemId); + + const save = new URL(actual.searchParams.get('save')!, CONFIG.baseURL); + expect(save.pathname).toBe(`/${CONFIG.apiRoot}/folder/${g.folderId}/volview`); + const linked = JSON.parse(save.searchParams.get('metadata') || '{}').linkedResources || {}; + expect(linked.items).toEqual([g.itemId]); + expect(linked.folders || []).toEqual([]); + }); + + test('checking a saved session in Girder opens that session', async ({ page, context }) => { + const g = await setupFixture(context, 'lifecycle-session-row'); + const checked = await launchGesture(page, g, 'checked'); + const resumeUrl = await remoteSave(checked.view); + expect(resumeUrl).toBeTruthy(); + const sessionId = resumeUrl.split('/item/')[1].split('/volview')[0]; + await checked.view.close(); + + await loginViaUI(page); + await gotoFolder(page, g.folderId); + await checkRowByItemId(page, sessionId); + await checkRowByItemId(page, g.itemId); + + const popupPromise = page.waitForEvent('popup'); + await page.locator('.open-in-volview').click(); + await expect(page.locator('.modal-content')).toContainText('Will open newest VolView session'); + await page.locator('#g-confirm-button').click(); + const popup = await popupPromise; + await popup.waitForLoadState('domcontentloaded'); + await waitForVolViewReady(popup); + expect(urlsParam(popup)).toContain(`items=${sessionId}`); + }); +}); diff --git a/e2e/tsconfig.json b/e2e/tsconfig.json new file mode 100644 index 0000000..883b948 --- /dev/null +++ b/e2e/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2021", + "module": "commonjs", + "moduleResolution": "node", + "lib": ["ES2021", "DOM"], + "types": ["node"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": [ + "tests/**/*.ts", + "helpers/**/*.ts", + "playwright.config.ts", + "compat.setup.ts", + "compat.teardown.ts" + ] +} diff --git a/girder_volview/__init__.py b/girder_volview/__init__.py index f032bdc..9a04921 100644 --- a/girder_volview/__init__.py +++ b/girder_volview/__init__.py @@ -1,8 +1,6 @@ import cherrypy -import errno -import copy -from girder import plugin, events +from girder import plugin from girder.api.describe import Description, autoDescribeRoute from girder.api import access from girder.api.rest import ( @@ -10,73 +8,38 @@ setResponseHeader, setContentDisposition, ) -from girder.constants import AccessType, TokenScope, SortDir +from girder.constants import AccessType, TokenScope from girder.models.file import File -from girder.models.upload import Upload -from girder.utility import RequestBodyStream -from girder.exceptions import GirderException, RestException - from girder.models.item import Item -from girder.utility import ziputil -# used by get config -import yaml -from girder.models.setting import Setting from girder.models.folder import Folder -from girder import logger -from girder.models.group import Group # server settings (from girder.cfg file probably) for proxiable endpoint below from girder.utility import config from .dicom import setupEventHandlers -from .utils import ( - SESSION_ZIP_EXTENSION, - isSessionItem, - isLoadableImage, - isLoadableFile, - filesToManifest, - singleVolViewZipOrImageFiles, - idStringToIdList, - normalizeLinkedResources, - loadModels, - getFiles, - findNewestSession, - getLinkedResources, - matchesSelectionSet, - getNewestDoc, - getTouchedTime, - getFilteredFiles, - getFilteredSessionFile, - sessionNameFromFilter, +from .backend import addBackendRoutes +from .backend.launch import ( + downloadManifest, + downloadResourceManifest, + getFolderConfigFile, + saveToItem, + saveToFolder, ) - -LARGE_IMAGE_CONFIG_FOLDER = "large_image.config_folder" - - -BASE_CONFIG = { - "io": {"segmentGroupExtension": "seg", "segmentGroupSaveFormat": "nii.gz", "layerExtension": "layer"}, - "disabledViewTypes": ["3D", "Oblique"], - "layouts": { - "Axial Coronal Sagittal": { - "direction": "row", - "items": [ - "axial", - { - "direction": "column", - "items": ["coronal", "sagittal"] - } - ] - }, - "Axial Only": [["axial"]], - }, -} +from .utils import isLoadableImage, isSessionFile def hasLoadableFile(files, user=None): + # Mirror what the launch manifest would actually resolve: a session file + # opens through (restore), and anything else must be a loadable image that + # is not working data (job outputs, transient staged inputs) — otherwise + # the Open-in-VolView button would launch an empty viewer. + itemCache = {} + folderCache = {} for fileEntry in files: - if isLoadableFile(fileEntry[1] if isinstance(fileEntry, tuple) else fileEntry, user): + file = fileEntry[1] if isinstance(fileEntry, tuple) else fileEntry + if isSessionFile(file) or isLoadableImage(file, user, itemCache, folderCache): return True return False @@ -118,192 +81,70 @@ def volViewLoadableFolder(self, folder): # iterator that yields a tuple of (path, file). The aggregation is much # faster, as it only takes one database roundtrip, rather than one per # folder and one per item. - files = Folder().collection.aggregate([ - {"$match": {"_id": folder["_id"]}}, - {"$graphLookup": { - "from": "folder", - "startWith": folder["_id"], - "connectFromField": "_id", - "connectToField": "parentId", - "as": "__children" - }}, - {"$lookup": { - "from": "folder", - "localField": "_id", - "foreignField": "_id", - "as": "__self", - }}, - {"$project": {"__children": {"$concatArrays": [ - "$__self", "$__children", - ]}}}, - {"$unwind": {"path": "$__children"}}, - {"$replaceRoot": {"newRoot": "$__children"}}, - {"$match": Folder().permissionClauses(self.getCurrentUser(), level=AccessType.READ)}, - {"$lookup": { - "from": "item", - "let": {"fid": "$_id"}, - "pipeline": [ - {"$match": {"$expr": {"$eq": ["$$fid", "$folderId"]}}}, - {"$project": {"_id": 1}}, - ], - "as": "__items", - }}, - {"$lookup": { - "from": "file", - "localField": "__items._id", - "foreignField": "itemId", - "as": "__files", - }}, - {"$unwind": "$__files"}, - {"$replaceRoot": {"newRoot": "$__files"}}, - ]) + files = Folder().collection.aggregate( + [ + {"$match": {"_id": folder["_id"]}}, + { + "$graphLookup": { + "from": "folder", + "startWith": folder["_id"], + "connectFromField": "_id", + "connectToField": "parentId", + "as": "__children", + } + }, + { + "$lookup": { + "from": "folder", + "localField": "_id", + "foreignField": "_id", + "as": "__self", + } + }, + { + "$project": { + "__children": { + "$concatArrays": [ + "$__self", + "$__children", + ] + } + } + }, + {"$unwind": {"path": "$__children"}}, + {"$replaceRoot": {"newRoot": "$__children"}}, + { + "$match": Folder().permissionClauses( + self.getCurrentUser(), level=AccessType.READ + ) + }, + { + "$lookup": { + "from": "item", + "let": {"fid": "$_id"}, + "pipeline": [ + {"$match": {"$expr": {"$eq": ["$$fid", "$folderId"]}}}, + {"$project": {"_id": 1}}, + ], + "as": "__items", + } + }, + { + "$lookup": { + "from": "file", + "localField": "__items._id", + "foreignField": "itemId", + "as": "__files", + } + }, + {"$unwind": "$__files"}, + {"$replaceRoot": {"newRoot": "$__files"}}, + ] + ) loadable = hasLoadableFile(files, user=self.getCurrentUser()) return {"loadable": loadable} -def uploadSession(model, parentId, user, size, metadata=None): - # modified from girder.api.v1.file.File.initUpload - parentType = model.__name__.lower() - name = f"session{SESSION_ZIP_EXTENSION}" - try: - # Metadata comes from the client; don't fail the save if the shape - # isn't what we expect. - linkedFilter = (metadata or {}).get("linkedResources", {}).get("filter") - name = sessionNameFromFilter(linkedFilter, SESSION_ZIP_EXTENSION) - except Exception: - pass - - mimeType = "application/zip" - reference = None - parent = model().load(id=parentId, user=user, level=AccessType.WRITE, exc=True) - - chunk = None - ct = cherrypy.request.body.content_type.value - if ( - ct not in cherrypy.request.body.processors - and ct.split("/", 1)[0] not in cherrypy.request.body.processors - ): - chunk = RequestBodyStream(cherrypy.request.body) - if chunk is not None and chunk.getSize() <= 0: - chunk = None - - try: - upload = Upload().createUpload( - user=user, - name=name, - parentType=parentType, - parent=parent, - size=size, - mimeType=mimeType, - reference=reference, - ) - except OSError as exc: - if exc.errno == errno.EACCES: - raise GirderException( - "Failed to create upload.", - f"girder.api.v1.{parentType}.volview_save", - ) - raise - if upload["size"] > 0: - if chunk: - return Upload().handleChunk(upload, chunk, filter=True, user=user) - - return upload - else: - return File().filter(Upload().finalizeUpload(upload), user) - - -@access.public(cookie=True, scope=TokenScope.DATA_WRITE) -@boundHandler -@autoDescribeRoute( - Description("Save VolView session in an item") - .param("itemId", "The item ID", paramType="path") - .errorResponse() -) -def saveToItem(self, itemId): - size = int(cherrypy.request.headers.get("Content-Length")) - if size == 0: - raise GirderException( - "Expected non-zero Content-Length header", "girder.api.v1.item.save-volview" - ) - - return uploadSession(Item, itemId, self.getCurrentUser(), size) - - -@access.public(cookie=True, scope=TokenScope.DATA_WRITE) -@boundHandler -@autoDescribeRoute( - Description("Save VolView session in an folder") - .param("folderId", "The folder ID", paramType="path") - .jsonParam( - "metadata", - "A JSON object containing the metadata keys to add to the item.", - ) - .errorResponse() -) -def saveToFolder(self, folderId, metadata): - user = self.getCurrentUser() - size = int(cherrypy.request.headers.get("Content-Length")) - if size == 0: - raise GirderException( - "Expected non-zero Content-Length header", - "girder.api.v1.folder.volview_save", - ) - # TODO: Should we check if the folder already has an item with a session, - # and, if so, upload to that item rather than calling upload on the - # folder? - fileDic = uploadSession(Folder, folderId, user, size, metadata) - # Ensure next downloadResourcesManifest request for will find this - # session.volview.zip as the freshest session that matches the selection set: - # If there are session.volview.zip items in linked items, - # use its metadata.linkedResources as this item's metadata.linkedResources - linkedResources = metadata["linkedResources"] - linkedItems = normalizeLinkedResources(linkedResources)["items"] - selectedItems = loadModels(user, Item, linkedItems) - newestSelectedSession = findNewestSession(selectedItems) - if newestSelectedSession: - # LinkedResources points to volview.zip. Change saved volview.zip linkedResources - # to match selected linkedResources so we find most recent volview.zip next manifest - # request for those linkedResources. - metadata = {"linkedResources": getLinkedResources(newestSelectedSession)} - - item = Item().load(fileDic["itemId"], user=user, level=AccessType.WRITE, exc=True) - Item().setMetadata(item, metadata) - return fileDic - - -# Deprecated, use downloadManifest -@access.public(cookie=True, scope=TokenScope.DATA_READ) -@boundHandler -@autoDescribeRoute( - Description("Download zip of item files that do not end in volview.zip") - .modelParam("itemId", model=Item, level=AccessType.READ) - .produces(["application/zip"]) - .errorResponse("ID was invalid.") - .errorResponse("Read access was denied for the item.", 403) -) -def downloadDatasets(self, item): - setResponseHeader("Content-Type", "application/zip") - setContentDisposition(item["name"] + ".zip") - - def stream(): - zip = ziputil.ZipGenerator(item["name"]) - sansSessions = [ - fileEntry - for fileEntry in Item().fileList(item, subpath=False, data=False) - if isLoadableImage(fileEntry[1], user=self.getCurrentUser()) - ] - toZip = [ - (path, File().download(file, headers=False)) for path, file in sansSessions - ] - for path, file in toZip: - for data in zip.addFile(file, path): - yield data - yield zip.footer() - - return stream - - @access.public(scope=TokenScope.DATA_READ, cookie=True) @boundHandler @autoDescribeRoute( @@ -336,8 +177,16 @@ def downloadProxiableFile(self, file, name): offset = 0 endByte = None - # to get s3_assetstore_adapter to proxy s3, we set headers to False, - # but to have a correct partial response, fill in headers and status code + # to get s3_assetstore_adapter to proxy s3, we set headers to False, but that + # also suppresses Girder's default download headers. Set safe ones explicitly + # so a proxied file always downloads (attachment) with an inert content type + # and can never render inline in a browser. Transparent to the engine's fetch, + # which reads the response body regardless of these headers. + if proxyRequest: + setResponseHeader("Content-Type", "application/octet-stream") + setContentDisposition(file["name"]) + + # to have a correct partial response, fill in headers and status code if proxyRequest and ( offset > 0 or (endByte is not None and endByte < file["size"]) ): @@ -347,7 +196,7 @@ def downloadProxiableFile(self, file, name): endByte = file["size"] # endByte is non-inclusive, so set Content-Range accordingly cherrypy.response.headers["Content-Range"] = ( - f"bytes {offset}-{endByte-1}/{file['size']}" + f"bytes {offset}-{endByte - 1}/{file['size']}" ) cherrypy.response.headers["Content-Length"] = str(endByte - offset) elif proxyRequest: @@ -359,264 +208,12 @@ def downloadProxiableFile(self, file, name): ) -@access.public(cookie=True, scope=TokenScope.DATA_READ) -@boundHandler -@autoDescribeRoute( - Description( - "Download JSON listing item file download URIs that do not end in volview.zip" - ) - .modelParam("itemId", model=Item, level=AccessType.READ) - .produces(["application/json"]) - .errorResponse("ID was invalid.") - .errorResponse("Read access was denied for the item.", 403) -) -def downloadManifest(self, item): - allFiles = list(Item().fileList(item, subpath=False, data=False)) - files = singleVolViewZipOrImageFiles(allFiles, user=self.getCurrentUser()) - return filesToManifest(files, item["folderId"]) - - -@access.public(cookie=True, scope=TokenScope.DATA_READ) -@boundHandler -@autoDescribeRoute( - Description("Download JSON with file download URIs") - .modelParam("folderId", model=Folder, level=AccessType.READ) - .param("folders", "Folder IDs.", required=False) - .param("items", "Item IDs.", required=False) - .jsonParam("filters", "Filter (dict) or filter list (array of dicts) to apply within a folder.", required=False) - .produces(["application/json"]) - .errorResponse("ID was invalid.") - .errorResponse("Read access was denied for the folders or items.", 403) -) -def downloadResourceManifest(self, folder, folders, items, filters): - user = self.getCurrentUser() - folders = idStringToIdList(folders or '') - items = idStringToIdList(items or '') - # filters is either a dict, a list of dicts, or absent. Anything else - # (bare scalar, null parsed to None already short-circuits via `if not - # filters`) gets rejected here rather than 500ing in Mongo. - if filters is not None and not isinstance(filters, (dict, list)): - raise RestException("filters must be a JSON object or array of objects") - files = [] - if not folders and not items: - if not filters: - # All files in folder (unless volview.zip is found as direct child) - filesInFolder = [ - fileEntry - for fileEntry in Folder().fileList(folder, subpath=False, data=False) - ] - files = singleVolViewZipOrImageFiles( - filesInFolder, user=user, includeFilterLinkedSessions=False, - ) - else: - files = getFilteredSessionFile(folder, filters, user) - if files is None: - filesInFolder = getFilteredFiles(folder, filters) - files = [(None, file) for file in filesInFolder] - else: - # else load selected files - selectedItems = loadModels(user, Item, items) - # If any selected items are session.volview.zip, - # find freshest and change selection set to match its linkedResources. - newestSelectedSession = findNewestSession(selectedItems) - if newestSelectedSession: - linkedResources = getLinkedResources(newestSelectedSession) - linkedFilter = linkedResources.get("filter") - if linkedFilter: - files = getFilteredSessionFile(folder, linkedFilter, user) - if files is None: - files = singleVolViewZipOrImageFiles( - Item().fileList(newestSelectedSession, subpath=False, data=False), - user=user, - ) - return filesToManifest(files, folder["_id"]) - folders = linkedResources["folders"] - items = linkedResources["items"] - - # Find session.volview.zips that match selection set - sessionItems = [ - item for item in Folder().childItems(folder) if isSessionItem(item) - ] - matchingSessionItems = [ - session - for session in sessionItems - if matchesSelectionSet(folders, items, session) - ] - latestSession = getNewestDoc(matchingSessionItems) - # compare touched time of session with max touched time of selected items/folders - selectedFolders = loadModels(user, Folder, folders) - latestSelectedDoc = getNewestDoc(selectedFolders + selectedItems) - if ( - latestSession - and latestSelectedDoc - and getTouchedTime(latestSession) >= getTouchedTime(latestSelectedDoc) - ): - # session touched time is newer than selected items/folders so load it - files = singleVolViewZipOrImageFiles( - Item().fileList(latestSession, subpath=False, data=False), user=user, - ) - else: - # Load selected folders and items excluding child session.volview.zip and .volview_config.yaml - files = getFiles(Folder, selectedFolders) + getFiles(Item, selectedItems) - files = [file for file in files if isLoadableImage(file[1], user)] - return filesToManifest(files, folder["_id"]) - - -def _mergeDictionaries(a, b): - """ - Merge two dictionaries recursively. If the second dictionary (or any - sub-dictionary) has a special key, value of '__all__': True, the updated - dictionary only contains values from the second dictionary and excludes - the __all__ key. - - :param a: the first dictionary. Modified. - :param b: the second dictionary that gets added to the first. - :returns: the modified first dictionary. - """ - if b.get("__all__") is True: - a.clear() - for key in b: - if isinstance(a.get(key), dict) and isinstance(b[key], dict): - _mergeDictionaries(a[key], b[key]) - elif key != "__all__" or b[key] is not True: - a[key] = b[key] - return a - - -def adjustConfigForUser(config, user): - """ - Given the current user, adjust the config so that only relevant and - combined values are used. If the root of the config dictionary contains - "access": {"user": , "admin": }, the base values are updated - based on the user's access level. If the root of the config contains - "group": {: , ...}, the base values are updated for - every group the user is a part of. - - The order of update is groups in C-sort alphabetical order followed by - access/user and then access/admin as they apply. - - :param config: a config dictionary. - """ - if not isinstance(config, dict): - return config - if isinstance(config.get("groups"), dict): - groups = config.pop("groups") - if user: - for group in Group().find( - {"_id": {"$in": user["groups"]}}, sort=[("name", SortDir.ASCENDING)] - ): - if isinstance(groups.get(group["name"]), dict): - config = _mergeDictionaries(config, groups[group["name"]]) - if isinstance(config.get("access"), dict): - accessList = config.pop("access") - if user and isinstance(accessList.get("user"), dict): - config = _mergeDictionaries(config, accessList["user"]) - if user and user.get("admin") and isinstance(accessList.get("admin"), dict): - config = _mergeDictionaries(config, accessList["admin"]) - return config - - -# Modified from https://github.com/girder/large_image/blob/aa1dc05665944e87eb9cb8553085221fab16ae92/girder/girder_large_image/__init__.py#L434-L483 -def yamlConfigFile(folder, name, user, addConfig): - """ - Get a resolved named config file based on a folder and user. - - :param folder: a Girder folder model. - :param name: the name of the config file. - :param user: the user that the response if adjusted for. - :returns: either None if no config file, or a yaml record. - """ - last = False - while folder: - item = Item().findOne({"folderId": folder["_id"], "name": name}) - if item: - for file in Item().childFiles(item): - if file["size"] > 10 * 1024**2: - logger.info("Not loading %s -- too large" % file["name"]) - continue - with File().open(file) as fptr: - config = yaml.safe_load(fptr) - if isinstance(config, list) and len(config) == 1: - config = config[0] - # combine and adjust config values based on current user - if ( - isinstance(config, dict) - and "access" in config - or "group" in config - ): - config = adjustConfigForUser(config, user) - if addConfig and isinstance(config, dict): - config = _mergeDictionaries(config, addConfig) - if ( - not isinstance(config, dict) - or config.get("__inherit__") is not True - ): - return config - config.pop("__inherit__") - addConfig = config - if last: - break - if folder["parentCollection"] != "folder": - if folder["name"] != ".config": - folder = Folder().findOne( - { - "parentId": folder["parentId"], - "parentCollection": folder["parentCollection"], - "name": ".config", - } - ) - else: - last = "setting" - if not folder or last == "setting": - folderId = Setting().get(LARGE_IMAGE_CONFIG_FOLDER) - if not folderId: - break - folder = Folder().load(folderId, force=True) - last = True - else: - folder = Folder().load(folder["parentId"], user=user, level=AccessType.READ) - return addConfig - - -@access.public(cookie=True, scope=TokenScope.DATA_READ) -@boundHandler() -@autoDescribeRoute( - Description("Get a VolView config file.") - .notes( - "Wraps large image yaml_config endpoint and inserts more properties. " - "This walks up the chain of parent folders until the file is found. " - "If not found, the .config folder in the parent collection or user is " - "checked.\n\nAny yaml file can be returned. If the top-level is a " - 'dictionary and contains keys "access" or "groups" where those are ' - "dictionaries, the returned value will be modified based on the " - 'current user. The "groups" dictionary contains keys that are group ' - "names and values that update the main dictionary. All groups that " - "the user is a member of are merged in alphabetical order. If a key " - 'and value of "\\__all\\__": True exists, the replacement is total; ' - 'otherwise it is a merge. If the "access" dictionary exists, the ' - '"user" and "admin" subdictionaries are merged if a calling user is ' - "present and if the user is an admin, respectively (both get merged " - "for admins)." - ) - .modelParam("folderId", model=Folder, level=AccessType.READ) - .param("name", "The name of the file.", paramType="path") - .produces(["application/json"]) - .errorResponse() -) -def getFolderConfigFile(self, folder, name): - user = self.getCurrentUser() - baseConfig = copy.deepcopy(BASE_CONFIG) - config = yamlConfigFile(folder, name, user, None) or {} - config = _mergeDictionaries(baseConfig, config) - return config - - class GirderPlugin(plugin.GirderPlugin): DISPLAY_NAME = "VolView" CLIENT_SOURCE_PATH = "web_client" def load(self, info): - plugin.getPlugin('large_image').load(info) + plugin.getPlugin("large_image").load(info) setupEventHandlers() info["apiRoot"].item.route( @@ -629,15 +226,16 @@ def load(self, info): info["apiRoot"].folder.route( "GET", (":folderId", "volview"), downloadResourceManifest ) + # Session-zip save: item-scoped stuffs the zip into the item, + # folder-scoped creates a new session.volview.zip item. Each returns a + # resumeUrl the client repoints its urls= at, so a later F5 reloads the + # just-made save. + info["apiRoot"].item.route("POST", (":itemId", "volview"), saveToItem) + info["apiRoot"].folder.route("POST", (":folderId", "volview"), saveToFolder) info["apiRoot"].file.route( "GET", (":id", "proxiable", ":name"), downloadProxiableFile ) info["apiRoot"].folder.route( "GET", (":folderId", "volview_config", ":name"), getFolderConfigFile ) - info["apiRoot"].folder.route("POST", (":folderId", "volview"), saveToFolder) - info["apiRoot"].item.route("POST", (":itemId", "volview"), saveToItem) - # volview/datasets is deprecated. Use GET {folder|item}/volview instead. - info["apiRoot"].item.route( - "GET", (":itemId", "volview", "datasets"), downloadDatasets - ) + addBackendRoutes(info) diff --git a/girder_volview/backend/__init__.py b/girder_volview/backend/__init__.py new file mode 100644 index 0000000..175fb8b --- /dev/null +++ b/girder_volview/backend/__init__.py @@ -0,0 +1,3 @@ +from .routes import addBackendRoutes + +__all__ = ["addBackendRoutes"] diff --git a/girder_volview/backend/config.py b/girder_volview/backend/config.py new file mode 100644 index 0000000..4922d2f --- /dev/null +++ b/girder_volview/backend/config.py @@ -0,0 +1,57 @@ +"""Processing backend -- the per-launch provider-config block. + +The launch manifest injects this block so the client knows where to reach the +processing provider (``baseUrl`` / ``jobsBaseUrl``) for the folder it opened. +Both URLs derive from the runtime ``getApiRoot()`` so a non-default API mount +still resolves. +""" + +from girder.utility.server import getApiRoot + +# The mount segment of every processing route: the folder-tree routes +# (``/folder/:id/volview_processing/...``) and the folder-free job resource +# (``/volview_processing/...``, see routes.py ``_JobResource``). Shared with the +# route registration so the advertised URLs and the mounted routes cannot drift. +PROCESSING_ROUTE_NAME = "volview_processing" +PROCESSING_PROVIDER_ID_PREFIX = "girder-slicer-cli" + + +def processingProviderId(folderId): + """Return the stable provider identity advertised for a launch folder.""" + return "%s:%s" % (PROCESSING_PROVIDER_ID_PREFIX, folderId) + + +def buildProcessingConfigBlock(folder): + # The block advertises only where to reach the provider, never what is + # loaded: the client mints its own input refs from the on-screen volume's + # provenance. The client zod schema (`src/processing/config.ts` + # processingProviderConfig) reads only id/label/baseUrl/jobsBaseUrl/context + # and strips unknown keys, so an added field stays compatible. + # + # The provider ID is FOLDER-SCOPED and immutable: it carries the launch + # folder id so two folders open simultaneously register as two distinct + # providers (the client keys every job by (providerId, jobId)); a bare + # "girder-slicer-cli" would make both folders share one mutable identity. The + # label carries the folder name so the picker distinguishes them (fall back to + # bare "Jobs" when a folder document has no name). + # + # Both URLs are origin-relative and keyed off getApiRoot() -- the SAME mount + # utils.makeFileDownloadUrl and inputs._fileIdFromMintedUri use. Hardcoding + # "/api/v1" 404s every submit/status/results/stage call on a non-default API + # mount (e.g. /girder/api/v1). jobsBaseUrl is the folder-free root for the + # job-addressed routes (status/results/cancel), advertised explicitly so the + # client never string-surgeries the folder segment out of baseUrl. + folderName = folder.get("name") + return { + "providers": [ + { + "id": processingProviderId(folder["_id"]), + "label": "Jobs — %s" % folderName if folderName else "Jobs", + "baseUrl": ( + f"/{getApiRoot()}/folder/{folder['_id']}/{PROCESSING_ROUTE_NAME}" + ), + "jobsBaseUrl": f"/{getApiRoot()}/{PROCESSING_ROUTE_NAME}", + "context": {}, + } + ] + } diff --git a/girder_volview/backend/inputs.py b/girder_volview/backend/inputs.py new file mode 100644 index 0000000..2be69fb --- /dev/null +++ b/girder_volview/backend/inputs.py @@ -0,0 +1,401 @@ +"""Processing backend -- input resolution + transient staging lifecycle.""" + +import datetime + +from bson.objectid import ObjectId +from girder import logger +from girder.constants import AccessType +from girder.exceptions import AccessException, RestException +from girder.models.file import File +from girder.models.item import Item +from girder.models.upload import Upload + +# Module-object import (not ``from ... import Job``): call sites resolve +# ``girder_job.Job`` at call time, so tests may monkeypatch the class on +# ``girder_jobs.models.job`` and be seen here. +from girder_jobs.models import job as girder_job + +from ..handles import parseFileHandle +from ..utils import TRANSIENT_STAGED_META_KEY, isTransientStagedItem + +# Every submitted uri is a backend-minted, origin-relative +# ``//file//proxiable/`` (``utils.makeFileDownloadUrl``). +# Resolution recovers the file id from that exact shape and nothing else, then +# re-checks READ access under the submitting user. Type-agnostic: every input +# resolves through this one path — the backend never branches on ``type``. + + +def _fileIdFromMintedUri(uri): + """Recover the Girder file id from a backend-minted proxiable uri, or ``None``. + + Delegates to :func:`girder_volview.handles.parseFileHandle`, the ONE parse + site for the load-handle scheme and the exact mirror of the mint + (``handles.mintFileHandle`` / ``utils.makeFileDownloadUrl``). Returns + ``None`` for anything outside the backend's own scheme, so a foreign or + malformed string is rejected by the caller and never dereferenced. + """ + parsed = parseFileHandle(uri) + return parsed[0] if parsed else None + + +def resolveInputUrisToFiles(uris, user): + """Resolve a client-minted uri list to readable Girder files (fail closed). + + A uri that does not match the backend's mint is rejected 400 and never + fetched, and every recovered id is loaded with the submitting user's READ + permission, so possession of a (by-design recoverable) id is not itself a + capability. Validation runs over every uri first, then authorization, so a + malformed uri fails 400 ahead of an unreadable id's 403. + """ + if not isinstance(uris, list) or not uris: + raise RestException("Processing input value carries no uris", code=400) + fileIds = [] + for uri in uris: + fileId = _fileIdFromMintedUri(uri) + if fileId is None: + raise RestException( + "Processing input uri does not match this server's file scheme", + code=400, + ) + fileIds.append(fileId) + return _readableFilesInOrder(fileIds, user) + + +def readableFilesById(fileObjectIds, user, fields=None): + """Load the file docs whose parent item ``user`` can READ, batched. + + THE one ACL boundary for bulk file reads: a file is readable iff its parent + item is READable. Girder files inherit access through their parent item, and + ``File().load`` with a user+level runs a per-file ACL that falls back to + loading that parent -- ~2 Mongo queries EACH (≈600 for a 300-slice DICOM + series). Batched instead: one ``File().find`` for every id, then ONE + permission-filtered ``Item().findWithPermissions`` over the distinct parent + items. Returns ``{str(fileId): fileDoc}``; a missing file, missing parent, + or unreadable parent is simply absent, and lenient/strict handling stays + with the callers. + """ + fileDocs = list( + File().find(query={"_id": {"$in": list(fileObjectIds)}}, fields=fields) + ) + itemIds = {fileDoc.get("itemId") for fileDoc in fileDocs} + itemIds.discard(None) + if not itemIds: + return {} + readableItemIds = { + itemDoc["_id"] + for itemDoc in Item().findWithPermissions( + query={"_id": {"$in": list(itemIds)}}, + fields={"_id": 1}, + user=user, + level=AccessType.READ, + ) + } + return { + str(fileDoc["_id"]): fileDoc + for fileDoc in fileDocs + if fileDoc.get("itemId") in readableItemIds + } + + +def _readableFilesInOrder(fileIds, user): + """Load READ-authorized file docs for ``fileIds``, in input order, raising. + + A strict adapter over :func:`readableFilesById`: any unreadable id raises + ``AccessException``. ``_fileIdFromMintedUri`` already validated each id's + shape, so the ``ObjectId`` conversion cannot fail here. Ordering matches the + input ``fileIds`` (a comma-joined multi-file volume forwards ids + positionally), and a repeated id resolves to the same doc. + """ + filesById = readableFilesById([ObjectId(fileId) for fileId in fileIds], user) + files = [] + for fileId in fileIds: + fileDoc = filesById.get(fileId) + if fileDoc is None: + # Missing file, missing parent, or a parent the user cannot READ: + # possession of a (by-design recoverable) id is not a capability. + raise AccessException("Read access denied for file %s." % fileId) + files.append(fileDoc) + return files + + +def validateStagedDescriptor(descriptor, user): + """Validate a staged-resource descriptor end to end; return its name. + + Owns the whole descriptor schema — shape, the ``labelmap`` type + discriminator, and the reference image (own-scheme + ACL + durable) — so the + ``stageInput`` route stays transport + authorization only and a future + staged type extends this one validator. + """ + if set(descriptor) != {"type", "name", "referenceImage"}: + raise RestException("Malformed staged resource descriptor", code=400) + if descriptor.get("type") != "labelmap": + raise RestException("Staged resource type must be labelmap", code=400) + name = descriptor.get("name") + if not isinstance(name, str) or not name: + raise RestException("Staged resource name must not be empty", code=400) + referenceImage = descriptor.get("referenceImage") + if not isinstance(referenceImage, dict): + raise RestException("Staged labelmap requires a reference image", code=400) + if not set(referenceImage).issubset({"type", "format", "uris"}): + raise RestException("Malformed staged reference image", code=400) + if "format" in referenceImage and not isinstance(referenceImage["format"], str): + raise RestException("Malformed staged reference image format", code=400) + validateStagedReferenceImage(referenceImage, user) + return name + + +def validateStagedReferenceImage(referenceImage, user): + """Validate a staged labelmap's reference image (own-scheme + ACL + durable). + + Resolves the reference image's own-scheme uris to files under the caller's + READ permission — rejecting a malformed, foreign, or unauthorized reference + — and rejects a transient reference so a staged labelmap never binds to + ephemeral data. Validation only: no lineage is tracked. + """ + if not isinstance(referenceImage, dict) or referenceImage.get("type") != "image": + raise RestException("Staged labelmap requires a reference image", code=400) + fileDocs = resolveInputUrisToFiles(referenceImage.get("uris"), user) + itemIds = {fileDoc.get("itemId") for fileDoc in fileDocs} + itemIds.discard(None) + # ``resolveInputUrisToFiles`` already enforced READ on each file's parent item, + # so this read only inspects the transient marker: one batched find over the + # DISTINCT parent items replaces a per-file (and per-duplicate) ``Item().load``. + for item in Item().find(query={"_id": {"$in": list(itemIds)}}): + if _isTransientItem(item): + raise RestException( + "Staged labelmap requires a durable reference image", code=400 + ) + + +# ``stageInput`` (in ``routes.py``) lands client-held bytes in a fresh item in +# the launch folder's server-owned jobs container, tags it transient, and mints +# a proxiable download URI for it; from there a staged input resolves through +# the same own-scheme path as any other input. +# Ownership is per-job by construction (``copyStagedInputsIntoJobFolder``), so +# no job references a shared staged original. Cleanup is therefore split: the +# job deletes its own copies at terminal state, and the TTL sweep below ages +# out the originals, which have no job to clean them up. + +# Age after which an uploaded-but-never-submitted transient item is swept on the +# next staging call. Upload->submit is normally seconds; a day absorbs an +# interrupted session without cluttering folders across days. +_TRANSIENT_ORPHAN_TTL = datetime.timedelta(hours=24) + + +# One definition of the staging predicate, shared with the launch-manifest +# exclusion (``utils.isTransientStagedFile``); aliased so this module's call +# sites and test doubles keep their name. +_isTransientItem = isTransientStagedItem + + +def copyStagedInputsIntoJobFolder(params, resolvedInputFiles, user, outputFolder): + """Give the job its OWN copies of any staged (transient) inputs. + + Every transient staged item among a submission's bound inputs is copied + into the job's private folder and the CLI file-id params are rewritten + onto the copies, so the job references only resources it alone owns and two + jobs reusing one staged original can never delete each other's inputs. The + original stays covered by the TTL orphan sweep. Transience is decided by + the parent item's marker, never by ``type``. + + URI resolution (``resolveInputUrisToFiles``) already enforced READ on every + parent item moments ago, so the transient markers are read with ONE batched + find over the distinct parents rather than a per-item ACL'd load. A parent + that vanished in between (an orphan sweep or delete) raises 409 rather than + publishing a job against deleted file ids. ``Item().copyItem`` deep-copies + metadata, so a copy carries the transient marker and is cleaned up exactly + like any staged item. + + Returns ``(params, copiedItemIds)`` — the (possibly rewritten) params and + the copied item ids to record on the job for terminal cleanup. + """ + itemIds = { + fileDoc["itemId"] + for fileDocs in resolvedInputFiles.values() + for fileDoc in fileDocs + if (fileDoc or {}).get("itemId") + } + items = list(Item().find({"_id": {"$in": list(itemIds)}})) if itemIds else [] + if len(items) != len(itemIds): + # URI resolution ACL-loaded these parents moments ago, so a missing item + # means a concurrent delete won the race. Fail the submit rather than + # publish a job whose params reference deleted files. + raise RestException( + "A processing input was removed while the submission " + "was in progress; please resubmit", + code=409, + ) + fileIdRemap = {} + copiedItemIds = [] + for item in items: + if not _isTransientItem(item): + continue + copied = Item().copyItem(item, creator=user, folder=outputFolder) + copiedItemIds.append(str(copied["_id"])) + # Copied files preserve name/size/checksum; sorting both sides by that + # triple pairs each original with its copy regardless of the underlying + # cursor order. Girder permits same-named files in one item, so name + # alone could pair A with B's copy — with the full triple, files that + # still tie are byte-identical and any pairing is correct. copyItem + # duplicates every child file, so a length mismatch means the staged + # item's files changed between resolution and copy — the same race as + # the concurrent-delete guard above, and the same typed 409 rather than + # running the job against a partial input. + def pairingKey(f): + return (f.get("name", ""), f.get("size", 0), f.get("sha512") or "") + + originals = sorted(Item().childFiles(item), key=pairingKey) + copies = sorted(Item().childFiles(copied), key=pairingKey) + if len(originals) != len(copies): + raise RestException( + "A processing input changed while the submission " + "was in progress; please resubmit", + code=409, + ) + fileIdRemap.update( + { + str(orig["_id"]): str(cop["_id"]) + for orig, cop in zip(originals, copies, strict=True) + } + ) + if not fileIdRemap: + return params, copiedItemIds + params = dict(params) + for paramName, fileDocs in resolvedInputFiles.items(): + params[paramName] = ",".join( + fileIdRemap.get(str(fileDoc["_id"]), str(fileDoc["_id"])) + for fileDoc in fileDocs + ) + return params, copiedItemIds + + +# Girder jobs are user-owned, not folder-linked, so `listJobHistory` can only +# scope to a launch folder if the context is stamped ON the job at submit, as +# plain otherFields (queryable Mongo keys). The task id is backend association +# data the history summary does not expose. +_LAUNCH_FOLDER_FIELD = "volviewLaunchFolderId" # str(folder _id) — scope key +_TASK_ID_FIELD = "volviewTaskId" + + +def _removeTransientItems(itemIds): + """Delete transient input items by id (idempotent, best-effort).""" + for itemId in itemIds: + try: + item = Item().load(itemId, force=True) + if item: + Item().remove(item) + except Exception: + logger.exception("Failed to remove transient item %s", itemId) + + +def _cleanupTransientOnJobDone(event): + """Delete a job's transient staged inputs once it reaches a terminal state. + + Bound to ``jobs.job.update.after``. Idempotent: a re-fired terminal update + finds the items already gone and no-ops. A present, non-terminal in-memory + status short-circuits before any DB work, so the common progress/log tick + costs nothing. When the in-memory status is terminal or absent, the job is + reloaded from the database before reading the marker/status -- the event + carries the updater's *in-memory* job dict, which holds the marker only if + that updater happened to DB-load the job first. Reloading keeps cleanup + self-contained for any terminal updater (girder_worker, a manual cancel). + """ + + from .results import isTerminalStatus + + info = getattr(event, "info", None) + eventJob = info.get("job") if isinstance(info, dict) else None + if not isinstance(eventJob, dict): + return + # This handler fires on EVERY job update instance-wide (progress ticks, log + # appends). ``updateJob`` sets the new status ON the in-memory job dict + # before firing, so a present, non-terminal in-memory status is an + # authoritative "not settled yet" -- short-circuit before the DB reload the + # steady-state stream would otherwise pay on every tick. + inMemoryStatus = eventJob.get("status") + if inMemoryStatus is not None and not isTerminalStatus(inMemoryStatus): + return + # Reload the committed doc to read the marker/status self-containedly -- the + # event's in-memory job dict may carry neither. includeLog=False because + # only the marker + status are read; loading the unbounded log would + # re-materialize it out of Mongo on every tick. + job = girder_job.Job().load(eventJob.get("_id"), force=True, includeLog=False) + if not isinstance(job, dict): + return + transientItemIds = job.get(TRANSIENT_STAGED_META_KEY) + if not isinstance(transientItemIds, list) or not transientItemIds: + return + if not isTerminalStatus(job.get("status")): + return + _removeTransientItems(transientItemIds) + + +def _sweepOrphanTransients(folder, now=None): + """Age out stale transient items in ``folder`` (best-effort). + + Piggybacked on staging calls (an upload precedes its job, so job-end cleanup + never sees a never-submitted orphan). Keyed off ``item['created']`` because + the marker carries no timestamp; only items strictly older than + :data:`_TRANSIENT_ORPHAN_TTL` are candidates, so the item this same call is + about to create is never one. Age alone decides: no job ever depends on a + staged ORIGINAL — submission rewires the job onto its own private copies + (:func:`copyStagedInputsIntoJobFolder`), which live in the job's private + folder, not the staging folder this sweep scans. + """ + now = now or datetime.datetime.utcnow() + cutoff = now - _TRANSIENT_ORPHAN_TTL + query = { + "folderId": folder["_id"], + "meta.%s" % TRANSIENT_STAGED_META_KEY: True, + "created": {"$lt": cutoff}, + } + try: + stale = list(Item().find(query)) + except Exception: + logger.exception("Failed to query orphan transient items") + return + for item in stale: + try: + Item().remove(item) + except Exception: + logger.exception( + "Failed to sweep orphan transient item %s", item.get("_id") + ) + + +def _streamMultipartFileIntoItem(folder, user, part, name): + """Stream one parsed multipart file part into a fresh item under ``folder``. + + ``folder`` is the server-owned jobs container derived from the launch folder + that ``stageInput`` already loaded with WRITE access. Its ACL mirrors that + launch folder, so this transport helper deliberately does not add another + authorization boundary. + """ + stream = getattr(part, "file", None) + if stream is None: + raise RestException("Staging request carries no file part", code=400) + stream.seek(0, 2) + size = stream.tell() + stream.seek(0) + if size <= 0: + raise RestException("Staging file must not be empty", code=400) + return Upload().uploadFromFile( + stream, + size=size, + name=name, + parentType="folder", + parent=folder, + user=user, + mimeType="application/octet-stream", + ) + + +def _tagItemTransient(fileDoc): + """Tag a freshly-uploaded file's parent item transient; return the item.""" + itemId = fileDoc.get("itemId") + if not itemId: + return None + item = Item().load(itemId, force=True) + if item: + Item().setMetadata(item, {TRANSIENT_STAGED_META_KEY: True}) + return item diff --git a/girder_volview/backend/launch.py b/girder_volview/backend/launch.py new file mode 100644 index 0000000..12a0683 --- /dev/null +++ b/girder_volview/backend/launch.py @@ -0,0 +1,510 @@ +"""Launch / compose / config / save handlers for the ordinary VolView viewer. + +Each launch gesture has one meaning: raw checked picks ALWAYS open fresh; a +checked session item opens through to exactly that session; a filter gesture +resumes its newest matching session; a bare folder-open resumes the folder's +newest unfiltered session. A save returns a ``resumeUrl`` the client uses for +subsequent reloads (F5); the save target itself stays launch-provided, so +folder saves mint a new ``session.volview.zip`` item per save. +""" + +import copy +import errno + +import cherrypy +import yaml + +from girder import logger +from girder.api import access +from girder.api.describe import Description, autoDescribeRoute +from girder.api.rest import boundHandler +from girder.constants import AccessType, TokenScope, SortDir +from girder.exceptions import GirderException, RestException +from girder.models.file import File +from girder.models.folder import Folder +from girder.models.group import Group +from girder.models.item import Item +from girder.models.setting import Setting +from girder.models.upload import Upload +from girder.utility import RequestBodyStream +from girder.utility.server import getApiRoot + +from .config import buildProcessingConfigBlock +from ..utils import ( + SESSION_ZIP_EXTENSION, + isJobOutputFolderItem, + isLaunchFile, + isLoadableImage, + primeLoadableImageCaches, + filesToManifest, + singleVolViewZipOrImageFiles, + getFilteredFiles, + getFilteredSessionFile, + getFiles, + getLinkedResources, + idStringToIdList, + findNewestSession, + loadModels, + normalizeLinkedResources, + sessionNameFromFilter, +) + +LARGE_IMAGE_CONFIG_FOLDER = "large_image.config_folder" + +BASE_CONFIG = { + "io": { + "segmentGroupExtension": "seg", + "segmentGroupSaveFormat": "nii.gz", + "layerExtension": "layer", + }, + "disabledViewTypes": ["3D", "Oblique"], + "layouts": { + "Axial Coronal Sagittal": { + "direction": "row", + "items": [ + "axial", + {"direction": "column", "items": ["coronal", "sagittal"]}, + ], + }, + "Axial Only": [["axial"]], + }, +} + + +def uploadSession(model, parentId, user, size, metadata=None): + # modified from girder.api.v1.file.File.initUpload + parentType = model.__name__.lower() + name = f"session{SESSION_ZIP_EXTENSION}" + try: + # Metadata comes from the client; don't fail the save if the shape + # isn't what we expect. + linkedFilter = (metadata or {}).get("linkedResources", {}).get("filter") + name = sessionNameFromFilter(linkedFilter, SESSION_ZIP_EXTENSION) + except Exception: + pass + + mimeType = "application/zip" + reference = None + parent = model().load(id=parentId, user=user, level=AccessType.WRITE, exc=True) + + chunk = None + ct = cherrypy.request.body.content_type.value + if ( + ct not in cherrypy.request.body.processors + and ct.split("/", 1)[0] not in cherrypy.request.body.processors + ): + chunk = RequestBodyStream(cherrypy.request.body) + if chunk is not None and chunk.getSize() <= 0: + chunk = None + + try: + upload = Upload().createUpload( + user=user, + name=name, + parentType=parentType, + parent=parent, + size=size, + mimeType=mimeType, + reference=reference, + ) + except OSError as exc: + if exc.errno == errno.EACCES: + raise GirderException( + "Failed to create upload.", + f"girder.api.v1.{parentType}.volview_save", + ) from exc + raise + if upload["size"] > 0: + if chunk: + return Upload().handleChunk(upload, chunk, filter=True, user=user) + + return upload + else: + return File().filter(Upload().finalizeUpload(upload), user) + + +def _saveResponse(sessionItemId): + """The save response — a SINGLE field: the session's load URL. + + The VolView client stays opaque to Girder ids: it never learns the item id, + only the ``resumeUrl`` (``item/:id/volview``), which it repoints ONLY its + reload (``urls=``) at. So a later F5 reloads exactly this save, while the + save target (``save=``) stays launch-provided — a folder-scoped save mints + a new ``session.volview.zip`` item on every save. + """ + return {"resumeUrl": f"/{getApiRoot()}/item/{sessionItemId}/volview"} + + +def _uploadWholeSession(model, parentId, user, errorIdentifier, metadata=None): + """Upload the session zip in one shot; 400 unless it finalized into a File. + + Only a finalized File carries ``itemId``. A resumable/partial upload (a + processor content-type, or a body shorter than the declared Content-Length) + returns the raw Upload doc with no ``itemId``; the single-shot save contract + can't continue, so fail with a clean 400 rather than report a success F5 + would contradict by restoring the previous zip (or a KeyError -> 500 and an + orphaned item). + """ + try: + size = int(cherrypy.request.headers.get("Content-Length")) + except (TypeError, ValueError): + # Absent (e.g. Transfer-Encoding: chunked) or non-integer header: + # the same clean rejection as an empty body, not an int() 500. + size = 0 + if size == 0: + raise GirderException( + "Expected non-zero Content-Length header", errorIdentifier + ) + fileDic = uploadSession(model, parentId, user, size, metadata) + if "itemId" not in fileDic: + raise RestException( + "Session save must upload the whole zip in one request.", code=400 + ) + return fileDic + + +@access.public(cookie=True, scope=TokenScope.DATA_WRITE) +@boundHandler +@autoDescribeRoute( + Description("Save VolView session in an item") + .param("itemId", "The item ID", paramType="path") + .errorResponse() +) +def saveToItem(self, itemId): + _uploadWholeSession( + Item, itemId, self.getCurrentUser(), "girder.api.v1.item.save-volview" + ) + # The session file is stuffed into this same item, so its own manifest URL + # is both the save target and the F5 reload target. + return _saveResponse(itemId) + + +@access.public(cookie=True, scope=TokenScope.DATA_WRITE) +@boundHandler +@autoDescribeRoute( + Description("Save VolView session in an folder") + .param("folderId", "The folder ID", paramType="path") + .jsonParam( + "metadata", + "A JSON object containing the metadata keys to add to the item.", + ) + .errorResponse() +) +def saveToFolder(self, folderId, metadata): + user = self.getCurrentUser() + # jsonParam yields whatever the client sent, so `metadata` can be a list or + # scalar. Coerce before the upload -- same "don't fail the save on an + # unexpected shape" stance as uploadSession, and a guard placed after the + # upload would raise only once the zip is stored, 500ing on an orphan item. + if not isinstance(metadata, dict): + metadata = {} + # Rebase this save's linkedResources onto the newest already-saved session in + # the selection set: a save from a checked-session open would otherwise stamp + # linkedResources={items:[S]} instead of S's own lineage. Load-bearing for + # filter sessions — a save from a checked FILTER-session open must inherit + # the filter link so the filter row resumes this newest save and the bare + # folder-open keeps excluding it. + # + # Resolved BEFORE the upload: a malformed linkedResources (truthy non-object) + # or an unloadable id must 4xx with nothing stored, not raise once the zip has + # already finalized into a session item that a folder-open would then pick as + # the newest session to restore. + rawLinked = metadata.get("linkedResources") + linkedResources = normalizeLinkedResources( + rawLinked if isinstance(rawLinked, dict) else None + ) + selectedItems = loadModels(user, Item, linkedResources["items"]) + newestSelectedSession = findNewestSession(selectedItems) + savedMetadata = metadata + if newestSelectedSession: + savedMetadata = {"linkedResources": getLinkedResources(newestSelectedSession)} + + fileDic = _uploadWholeSession( + Folder, folderId, user, "girder.api.v1.folder.volview_save", metadata + ) + item = Item().load(fileDic["itemId"], user=user, level=AccessType.WRITE, exc=True) + try: + Item().setMetadata(item, savedMetadata) + except Exception: + # An unstamped session item is still the folder's newest session, so a + # later folder-open would restore this failed save. Drop it instead. + Item().remove(item) + raise + return _saveResponse(fileDic["itemId"]) + + +@access.public(cookie=True, scope=TokenScope.DATA_READ) +@boundHandler +@autoDescribeRoute( + Description( + "Download the VolView launch manifest for an item: a session.volview.zip " + "item opens through (restore); any other item resolves to its loadable " + "images (fresh)." + ) + .modelParam("itemId", model=Item, level=AccessType.READ) + .produces(["application/json"]) + .errorResponse("ID was invalid.") + .errorResponse("Read access was denied for the item.", 403) +) +def downloadManifest(self, item): + user = self.getCurrentUser() + # Job outputs stay durable in the folder but out of the launch manifest: a + # direct open of an item inside a job's private output folder yields nothing. + if isJobOutputFolderItem(item): + return filesToManifest([], item["folderId"]) + allFiles = list(Item().fileList(item, subpath=False, data=False)) + # A session file opens through (restore); otherwise the item's raw images. + files = singleVolViewZipOrImageFiles( + allFiles, user=user, itemCache={item["_id"]: item}, folderCache={} + ) + return filesToManifest(files, item["folderId"]) + + +@access.public(cookie=True, scope=TokenScope.DATA_READ) +@boundHandler +@autoDescribeRoute( + Description( + "Download the VolView launch manifest for a folder / checked / filter " + "gesture: a checked session item opens through to exactly that session " + "(back-in-history); raw checked items/folders ALWAYS open fresh; a " + "filter gesture resumes its newest matching session when one exists; a " + "bare folder-open resumes the folder's newest session.volview.zip, " + "else all its raw images. An explicit folders/items " + "selection takes precedence over filters; filters apply only when no " + "selection is passed, and the filtered leg returns only loadable images " + "(transient staged inputs and job-output-folder files are excluded)." + ) + .modelParam("folderId", model=Folder, level=AccessType.READ) + .param("folders", "Folder IDs.", required=False) + .param("items", "Item IDs.", required=False) + .jsonParam( + "filters", + "Filter (dict) or filter list (array of dicts) to apply within a folder.", + required=False, + ) + .produces(["application/json"]) + .errorResponse("ID was invalid.") + .errorResponse("Read access was denied for the folders or items.", 403) +) +def downloadResourceManifest(self, folder, folders, items, filters): + user = self.getCurrentUser() + itemCache = {} + folderCache = {} + folders = idStringToIdList(folders or "") + items = idStringToIdList(items or "") + # filters is either a dict, a list of dicts, or absent. Anything else + # (bare scalar) is rejected here rather than 500ing in Mongo. + if filters is not None and not isinstance(filters, (dict, list)): + raise RestException("filters must be a JSON object or array of objects") + # An explicit folders/items selection wins over filters: a stale/bookmarked + # URL carrying both must load the checked resources, not silently + # substitute the filter set. + if folders or items: + selectedItems = loadModels(user, Item, items) + checkedSession = findNewestSession(selectedItems) + if checkedSession: + # An explicitly checked session item opens through to EXACTLY that + # session — the back-in-history gesture. Never re-match it to a + # newer sibling save. Filter-linked sessions get the same treatment: + # re-entering the filter row resumes the newest, but checking an old + # one opens exactly it. + files = singleVolViewZipOrImageFiles( + Item().fileList(checkedSession, subpath=False, data=False), + user=user, + itemCache=itemCache, + folderCache=folderCache, + ) + return filesToManifest(files, folder["_id"]) + + # Raw checked picks ALWAYS open fresh: checking images is the "start + # fresh" gesture, so no saved session is ever substituted. Resume + # happens only through the other gestures — bare folder-open (newest + # save), checking a session item (exactly that save), or re-entering a + # filter row (newest filter save). + selectedFolders = loadModels(user, Folder, folders) + files = getFiles(Folder, selectedFolders) + getFiles(Item, selectedItems) + primeLoadableImageCaches([f[1] for f in files], user, itemCache, folderCache) + files = [ + f for f in files if isLoadableImage(f[1], user, itemCache, folderCache) + ] + elif filters: + files = getFilteredSessionFile(folder, filters, user) + # Empty is treated as no-match, not as a resolved session: a matched + # session whose files are all unloadable would otherwise skip the fresh + # leg and emit a manifest of nothing but config.json — a blank viewer + # with no gesture that recovers it. + if not files: + files = getFilteredFiles(folder, filters) + # The filter row owns every file it matched — no loadability gate + # (grouped DICOM rows carry extensionless slices). Only working + # data is excluded: transient staged inputs and session zips. + primeLoadableImageCaches(files, user, itemCache, folderCache) + files = [ + (None, f) + for f in files + if isLaunchFile(f, user, itemCache, folderCache) + ] + else: + # Bare folder-open -> resume the folder's newest session.volview.zip, + # else all its raw images. Filter-linked sessions are excluded (they are + # only meaningful re-entered through their filter). + filesInFolder = list(Folder().fileList(folder, subpath=False, data=False)) + files = singleVolViewZipOrImageFiles( + filesInFolder, + user=user, + includeFilterLinkedSessions=False, + itemCache=itemCache, + folderCache=folderCache, + ) + return filesToManifest(files, folder["_id"]) + + +def _mergeDictionaries(a, b): + """ + Merge two dictionaries recursively. If the second dictionary (or any + sub-dictionary) has a special key, value of '__all__': True, the updated + dictionary only contains values from the second dictionary and excludes + the __all__ key. + + :param a: the first dictionary. Modified. + :param b: the second dictionary that gets added to the first. + :returns: the modified first dictionary. + """ + if b.get("__all__") is True: + a.clear() + for key in b: + if isinstance(a.get(key), dict) and isinstance(b[key], dict): + _mergeDictionaries(a[key], b[key]) + elif key != "__all__" or b[key] is not True: + a[key] = b[key] + return a + + +def adjustConfigForUser(config, user): + """ + Given the current user, adjust the config so that only relevant and + combined values are used. If the root of the config dictionary contains + "access": {"user": , "admin": }, the base values are updated + based on the user's access level. If the root of the config contains + "group": {: , ...}, the base values are updated for + every group the user is a part of. + + The order of update is groups in C-sort alphabetical order followed by + access/user and then access/admin as they apply. + + :param config: a config dictionary. + """ + if not isinstance(config, dict): + return config + if isinstance(config.get("groups"), dict): + groups = config.pop("groups") + if user: + for group in Group().find( + {"_id": {"$in": user["groups"]}}, sort=[("name", SortDir.ASCENDING)] + ): + if isinstance(groups.get(group["name"]), dict): + config = _mergeDictionaries(config, groups[group["name"]]) + if isinstance(config.get("access"), dict): + accessList = config.pop("access") + if user and isinstance(accessList.get("user"), dict): + config = _mergeDictionaries(config, accessList["user"]) + if user and user.get("admin") and isinstance(accessList.get("admin"), dict): + config = _mergeDictionaries(config, accessList["admin"]) + return config + + +# Modified from https://github.com/girder/large_image/blob/aa1dc05665944e87eb9cb8553085221fab16ae92/girder/girder_large_image/__init__.py#L434-L483 +def yamlConfigFile(folder, name, user, addConfig): + """ + Get a resolved named config file based on a folder and user. + + :param folder: a Girder folder model. + :param name: the name of the config file. + :param user: the user that the response if adjusted for. + :returns: either None if no config file, or a yaml record. + """ + last = False + while folder: + item = Item().findOne({"folderId": folder["_id"], "name": name}) + if item: + for file in Item().childFiles(item): + if file["size"] > 10 * 1024**2: + logger.info("Not loading %s -- too large" % file["name"]) + continue + with File().open(file) as fptr: + config = yaml.safe_load(fptr) + if isinstance(config, list) and len(config) == 1: + config = config[0] + # combine and adjust config values based on current user + if isinstance(config, dict) and ( + "access" in config or "groups" in config + ): + config = adjustConfigForUser(config, user) + if addConfig and isinstance(config, dict): + config = _mergeDictionaries(config, addConfig) + if ( + not isinstance(config, dict) + or config.get("__inherit__") is not True + ): + return config + config.pop("__inherit__") + addConfig = config + if last: + break + if folder["parentCollection"] != "folder": + if folder["name"] != ".config": + folder = Folder().findOne( + { + "parentId": folder["parentId"], + "parentCollection": folder["parentCollection"], + "name": ".config", + } + ) + else: + last = "setting" + if not folder or last == "setting": + folderId = Setting().get(LARGE_IMAGE_CONFIG_FOLDER) + if not folderId: + break + folder = Folder().load(folderId, force=True) + last = True + else: + folder = Folder().load(folder["parentId"], user=user, level=AccessType.READ) + return addConfig + + +@access.public(cookie=True, scope=TokenScope.DATA_READ) +@boundHandler() +@autoDescribeRoute( + Description("Get a VolView config file.") + .notes( + "Wraps large image yaml_config endpoint and inserts more properties. " + "This walks up the chain of parent folders until the file is found. " + "If not found, the .config folder in the parent collection or user is " + "checked.\n\nAny yaml file can be returned. If the top-level is a " + 'dictionary and contains keys "access" or "groups" where those are ' + "dictionaries, the returned value will be modified based on the " + 'current user. The "groups" dictionary contains keys that are group ' + "names and values that update the main dictionary. All groups that " + "the user is a member of are merged in alphabetical order. If a key " + 'and value of "\\__all\\__": True exists, the replacement is total; ' + 'otherwise it is a merge. If the "access" dictionary exists, the ' + '"user" and "admin" subdictionaries are merged if a calling user is ' + "present and if the user is an admin, respectively (both get merged " + "for admins)." + ) + .modelParam("folderId", model=Folder, level=AccessType.READ) + .param("name", "The name of the file.", paramType="path") + .produces(["application/json"]) + .errorResponse() +) +def getFolderConfigFile(self, folder, name): + user = self.getCurrentUser() + baseConfig = copy.deepcopy(BASE_CONFIG) + config = yamlConfigFile(folder, name, user, None) or {} + config = _mergeDictionaries(baseConfig, config) + # Injected dynamically rather than living in BASE_CONFIG: the providers list + # depends on the folder being launched. + processing = buildProcessingConfigBlock(folder) + _mergeDictionaries(config, {"processing": processing}) + return config diff --git a/girder_volview/backend/outputs.py b/girder_volview/backend/outputs.py new file mode 100644 index 0000000..18155a0 --- /dev/null +++ b/girder_volview/backend/outputs.py @@ -0,0 +1,401 @@ +"""Processing backend -- folder-owned job-output correlation (write + cleanup). + +Each backend job OWNS exactly one server-created private output folder. Every +output ``girder_worker`` uploads is forced into that folder, so an upload is +correlated back to its job by the finalized file's ACTUAL parent folder -- never +by a filename, a token, or a caller-supplied job id (all of which are +attacker-controllable). The read path (projecting recorded ids into result +intents) and status projection live in ``results.py``. + +Ownership is also the deletion boundary, and it cascades both ways: a +``model.job.remove`` handler deletes the owned output folder plus any remaining +staged inputs and REFUSES a non-terminal owned job, while a +``model.folder.remove`` handler deletes the owning job when its output folder is +removed in the Girder hierarchy. Girder fires these handlers synchronously +BEFORE the DB delete and wraps them in no try/except, so raising aborts the +removal and retains the job record as the discoverable owner of whatever is left. +""" + +import json + +from girder.exceptions import RestException +from girder.models.folder import Folder +from girder.models.item import Item + +# Module-object import (not ``from ... import Job``): call sites resolve +# ``girder_job.Job`` at call time, so tests may monkeypatch the class on +# ``girder_jobs.models.job`` and be seen here. +from girder_jobs.models import job as girder_job + +from ..utils import JOB_OUTPUT_FOLDER_META_KEY, TRANSIENT_STAGED_META_KEY +from .inputs import _removeTransientItems + +# Backend-owned job fields (otherFields, not a schema change). The id map is +# READ-exposed (``routes.addBackendRoutes``); the job's own ACL is the gate. +_OUTPUTS_FIELD = "volviewOutputs" # {identifier: str(fileId)} +_OUTPUT_SPECS_FIELD = "volviewOutputSpecs" # [{name, tag, isLabel, fileExtensions}] +_OUTPUT_FOLDER_ID_FIELD = "volviewOutputFolderId" # str(folder _id) -- the job's +# private output folder; the SOLE output-correlation and ownership key. + + +def _parseOutputReference(raw): + """Decode an upload reference to a dict, or ``None``. + + ``girder_worker`` stamps each output upload with slicer_cli_web's JSON + reference (``prepare_task.py`` -- carries the output ``identifier``). A + non-JSON / non-dict / identifier-less reference (a foreign upload, or the + backend's own reference-less staging upload) yields ``None`` so the handler + skips it (fail closed). An identifier carrying ``.`` or ``$`` is rejected too, + so it can never be used to build an unintended nested / operator ``$set`` key. + """ + if raw is None: + return None + try: + ref = raw if isinstance(raw, dict) else json.loads(raw) + except (ValueError, TypeError): + return None + if not isinstance(ref, dict): + return None + identifier = ref.get("identifier") + if not isinstance(identifier, str) or not identifier: + return None + if "." in identifier or "$" in identifier: + return None + return ref + + +def _declaredOutputIdentifiers(job): + """The set of output identifiers the job DECLARED at submit (or empty). + + An upload whose identifier is not one this job declared is never recorded -- + a crafted reference cannot introduce an undeclared output key. + """ + specs = (job or {}).get(_OUTPUT_SPECS_FIELD) + if not isinstance(specs, list): + return set() + return { + spec["name"] + for spec in specs + if isinstance(spec, dict) and isinstance(spec.get("name"), str) + } + + +def _jobForOutputFolder(folderId): + """Find the job that OWNS a given output folder, or ``None`` (fail closed). + + Correlation is solely by the finalized file's actual parent folder: each job + owns exactly one private output folder, so the parent folder uniquely + identifies the job. A caller-supplied jobId / uuid / token / filename in the + upload reference is NEVER consulted. The id is stored as a string on the job, + so the query stringifies the parent folder id to match. + """ + + if folderId is None: + return None + job = girder_job.Job().findOne({_OUTPUT_FOLDER_ID_FIELD: str(folderId)}) + return job if isinstance(job, dict) else None + + +def _recordJobOutput(event): + """Synchronously record a finalized output file's id onto its owning job. + + Correlate the upload to a job by the finalized file's ACTUAL parent folder (a + File doc carries only ``itemId``, so this hops item -> folder), require the + reference ``identifier`` to be one the job DECLARED, and record the file id + keyed by that identifier (``otherFields`` dotted key -> Mongo ``$set`` nests + per identifier, so N outputs each bind without overwriting). Fail closed: a + foreign or uncorrelated upload, or an undeclared / unsafe identifier, is + ignored. This event fires synchronously before ``finalizeUpload`` returns, so + a DB failure while recording FAILS the upload (and the worker task) instead of + allowing a false SUCCESS with an unbound result. + """ + info = getattr(event, "info", None) + if not isinstance(info, dict): + return + upload = info.get("upload") + if not isinstance(upload, dict): + return + ref = _parseOutputReference(upload.get("reference")) + if ref is None: + return + fileDoc = info.get("file") + if not isinstance(fileDoc, dict): + return + fileId = fileDoc.get("_id") + itemId = fileDoc.get("itemId") + if not fileId or not itemId: + return + # A File doc has no ``folderId``; only its parent Item does. The item's parent + # folder is the job-correlation key. + item = Item().load(itemId, force=True, exc=False) + parentFolderId = item.get("folderId") if isinstance(item, dict) else None + if parentFolderId is None: + return + job = _jobForOutputFolder(parentFolderId) + if not isinstance(job, dict): + return + identifier = ref["identifier"] + if identifier not in _declaredOutputIdentifiers(job): + # An upload into the job's own folder whose identifier the job never + # declared is still refused -- correlation binds only declared outputs. + return + + girder_job.Job().updateJob( + job, + otherFields={"%s.%s" % (_OUTPUTS_FIELD, identifier): str(fileId)}, + ) + + +def _cascadeDeleteJobOwnedResources(event): + """``model.job.remove`` handler: enforce job/output ownership on deletion. + + For a job that owns an output folder (``_OUTPUT_FOLDER_ID_FIELD`` present): + + * REFUSE to remove a non-terminal job -- raise so Girder never reaches the + DB delete (this protects our own DELETE route AND Girder's built-in job + route and any direct ``Job.remove`` caller); + * otherwise cascade-delete the owned output folder (``Folder().remove`` + cascades to its items / subfolders / pending uploads) and then any + remaining staged input items; + * treat an already-missing owned folder as success (deletion is retryable); + * let a folder-removal failure PROPAGATE so the model retains the job record + -- the retained record stays the discoverable owner of whatever is left, + and a later retry completes the cascade. + + Jobs without the ownership field are untouched (standard removal proceeds). + """ + from .results import isTerminalStatus + + job = getattr(event, "info", None) + if not isinstance(job, dict): + return + folderId = job.get(_OUTPUT_FOLDER_ID_FIELD) + transientItemIds = job.get(TRANSIENT_STAGED_META_KEY) or [] + if not folderId and not transientItemIds: + return # not an owned job -- do not interfere with standard removal + if folderId: + if not isTerminalStatus(job.get("status")): + raise RestException( + "Cannot delete a job that is not finished", code=409 + ) + # The output folder is the critical owned resource (it holds the + # results); a failure here propagates so the job is retained and the + # delete is retryable. The in-progress marker (NOT a DB unset, which + # would break retryability on a partial failure) stops the + # model.folder.remove reverse cascade from re-entering Job.remove + # for this same job mid-delete. + folder = Folder().load(folderId, force=True, exc=False) + if folder is not None: + _CASCADING_FOLDER_IDS.add(str(folderId)) + try: + Folder().remove(folder) + finally: + _CASCADING_FOLDER_IDS.discard(str(folderId)) + # Staged inputs are transient (also orphan-swept); clean them best-effort so a + # stray input never blocks removing the (already-gone) results folder + job. + # Swept even when the folder pointer is absent: the reverse cascade unsets it + # before re-entering removal, and its jobs still own their staged inputs. + _removeTransientItems(transientItemIds) + + +# Output-folder ids currently being removed by the job-side cascade above. +# GIL-atomic set mutations; entries live only for the synchronous span of one +# Folder().remove call, so the reverse handler below can tell "the job is +# deleting its own folder" apart from "a user deleted the folder in Girder". +_CASCADING_FOLDER_IDS = set() + + +def _cascadeDeleteFolderOwnedJob(event): + """``model.folder.remove`` handler: removing a job's output folder removes the job. + + The reverse of ``_cascadeDeleteJobOwnedResources``: the private output + folder is the job's sole owned storage, so deleting it in the Girder + hierarchy means "delete this job". Removing the ``volview-jobs`` container + recurses through the per-job subfolders, firing this handler once per job. + + Mirrored invariants: + + * REFUSE to remove a live job's folder -- a non-terminal owned job raises, + aborting the folder removal (same guard as the job-side cascade); + * recursion guard -- unset ``volviewOutputFolderId`` on the job (DB + the + in-memory doc) BEFORE ``girder_job.Job().remove``, so the job-side cascade sees + no owned folder and only sweeps staged inputs. Unsetting is safe in this + direction: the folder is going away regardless, so a retained pointer + could only dangle. + + No-ops fail closed: an unmarked folder, a folder no job owns (the container, + a pre-publication orphan, an already-cascaded delete), or a removal driven + by the job-side cascade itself (``_CASCADING_FOLDER_IDS``). + """ + + from .results import isTerminalStatus + + folder = getattr(event, "info", None) + if not isinstance(folder, dict): + return + if not (folder.get("meta") or {}).get(JOB_OUTPUT_FOLDER_META_KEY): + return + folderId = folder.get("_id") + if folderId is None or str(folderId) in _CASCADING_FOLDER_IDS: + return + job = _jobForOutputFolder(folderId) + if job is None: + return + if not isTerminalStatus(job.get("status")): + raise RestException( + "Cannot delete the output folder of a job that is not finished; " + "cancel the job first", + code=409, + ) + girder_job.Job().update( + {"_id": job["_id"]}, {"$unset": {_OUTPUT_FOLDER_ID_FIELD: ""}} + ) + job.pop(_OUTPUT_FOLDER_ID_FIELD, None) + try: + girder_job.Job().remove(job) + except Exception: + # Restore the ownership pointer so a failed job removal never orphans + # the history row: the raise aborts the folder delete (the shell is + # retained), and the restored pointer keeps the job correlated with it + # for a later retry. + girder_job.Job().update( + {"_id": job["_id"]}, + {"$set": {_OUTPUT_FOLDER_ID_FIELD: str(folderId)}}, + ) + job[_OUTPUT_FOLDER_ID_FIELD] = str(folderId) + raise + + +def _folderChainMatchesTargets(folderId, folderTargets, baseTargets): + """Whether ``folderId`` is/descends from any folder in ``folderTargets``, + or its ROOT folder hangs directly under any ``(parentType, id)`` pair in + ``baseTargets`` (a collection or user about to be recursively deleted). + + Walks the folder's parent chain upward (owned output folders nest a couple + of levels deep, so the chain is short). Fail closed on a missing folder; + a (corrupt) parent cycle terminates via ``seen``. + """ + seen = set() + currentId = folderId + while currentId is not None and str(currentId) not in seen: + if str(currentId) in folderTargets: + return True + seen.add(str(currentId)) + folder = Folder().load(currentId, force=True, exc=False) + if not isinstance(folder, dict): + return False + if folder.get("parentCollection") != "folder": + return ( + folder.get("parentCollection"), + str(folder.get("parentId")), + ) in baseTargets + currentId = folder.get("parentId") + return False + + +def _liveJobOwningFolderUnderTargets(folderIds=(), baseParents=()): + """The first non-terminal job whose owned output folder is one of + ``folderIds``, a descendant of one, or contained (transitively) in any + ``(parentType, id)`` collection/user in ``baseParents`` — else ``None``. + + Persisted job ownership is the AUTHORITATIVE record: neither the deleted + folder nor anything on the path needs to carry the folder marker, so + deleting an unmarked ancestor (the launch folder, a project root) or a + folder whose marker was stripped cannot bypass the guard. Live owned jobs + are few at any moment, so checking each one's ancestor chain is cheap; the + query excludes settled history via the indexed ownership field + status. + """ + + from .results import terminalStatuses + + folderTargets = {str(folderId) for folderId in folderIds} + baseTargets = {(parentType, str(_id)) for parentType, _id in baseParents} + if not folderTargets and not baseTargets: + return None + jobs = girder_job.Job().find( + { + _OUTPUT_FOLDER_ID_FIELD: {"$exists": True}, + "status": {"$nin": list(terminalStatuses())}, + } + ) + for job in jobs: + if _folderChainMatchesTargets( + job.get(_OUTPUT_FOLDER_ID_FIELD), folderTargets, baseTargets + ): + return job + return None + + +def _refuseIfLiveJobUnder(folderIds=(), baseParents=()): + if _liveJobOwningFolderUnderTargets(folderIds, baseParents) is not None: + raise RestException( + "Cannot delete a processing job's output folder while the job is " + "still running; cancel the job first", + code=409, + ) + + +def _refuseLiveJobFolderRestDelete(event): + """``rest.delete.folder/:id.before`` guard: 409 a live job's folder delete. + + ``Folder.remove`` deletes contents (``clean``) BEFORE ``model.folder.remove`` + fires, so the reverse-cascade handler's non-terminal refusal can only save + the folder shell, not what was inside. This REST-level guard runs before the + delete handler touches anything, covering a live job's own folder, the + ``volview-jobs`` container, and ANY ancestor of either (deleting the launch + folder or a project root recursively cleans the job folder just the same). + Direct model callers bypass REST and still hit the (shell-level) late guard. + """ + info = getattr(event, "info", None) + folderId = (info or {}).get("id") + if folderId: + _refuseIfLiveJobUnder(folderIds=[folderId]) + + +def _refuseLiveJobCollectionRestDelete(event): + """``rest.delete.collection/:id.before``: the same preflight for collection + deletion, which recursively removes every folder inside without ever + passing through ``DELETE /folder/:id``.""" + info = getattr(event, "info", None) + collectionId = (info or {}).get("id") + if collectionId: + _refuseIfLiveJobUnder(baseParents=[("collection", collectionId)]) + + +def _refuseLiveJobUserRestDelete(event): + """``rest.delete.user/:id.before``: the same preflight for user deletion + (removes the user's whole folder tree).""" + info = getattr(event, "info", None) + userId = (info or {}).get("id") + if userId: + _refuseIfLiveJobUnder(baseParents=[("user", userId)]) + + +def _refuseLiveJobResourceRestDelete(event): + """``rest.delete.resource.before``: the same preflight for the batch + ``DELETE /resource`` route (arbitrary folder/collection/user ids). + + A malformed ``resources`` payload is left for the route itself to 400; + item ids are ignored (an item cannot contain a job's output folder). + """ + info = getattr(event, "info", None) + raw = ((info or {}).get("params") or {}).get("resources") + try: + resources = json.loads(raw) if isinstance(raw, str) else raw + except ValueError: + return + if not isinstance(resources, dict): + return + ids = { + model: [_id for _id in values if _id] + for model, values in resources.items() + if isinstance(values, list) + } + _refuseIfLiveJobUnder( + folderIds=ids.get("folder", ()), + baseParents=[ + (parentType, _id) + for parentType in ("collection", "user") + for _id in ids.get(parentType, ()) + ], + ) diff --git a/girder_volview/backend/results.py b/girder_volview/backend/results.py new file mode 100644 index 0000000..9882302 --- /dev/null +++ b/girder_volview/backend/results.py @@ -0,0 +1,427 @@ +"""Processing backend -- job status projection + result collection (the read path). + +Girder ``JobStatus`` projects to the contract's neutral shapes (never the girder +enum on the wire). Results come from the file ids recorded ON the job by +``outputs._recordJobOutput`` -- reference-bound, never a folder-name scan. +""" + +import functools + +from bson.objectid import ObjectId +from girder import logger +from girder_jobs.constants import JobStatus + +from ..utils import makeFileDownloadUrl, _toIso +from .config import processingProviderId +from .inputs import _LAUNCH_FOLDER_FIELD, _TASK_ID_FIELD, readableFilesById +from .outputs import ( + _OUTPUTS_FIELD, + _OUTPUT_SPECS_FIELD, + _declaredOutputIdentifiers, +) + + +@functools.cache +def _workerActiveStates(): + """girder_worker ``CustomJobStatus`` active-state codes (with numeric fallback). + + Core's ``JobStatus`` map has no entry for these, so without them a running job + would default to ``"pending"`` and a polling client would see it REGRESS. The + numeric literals are the stable wire integers used when girder_worker (an + optional runtime dependency) is not importable. + """ + try: + from girder_worker.utils import CustomJobStatus + + return { + CustomJobStatus.FETCHING_INPUT, + CustomJobStatus.CONVERTING_INPUT, + CustomJobStatus.CONVERTING_OUTPUT, + CustomJobStatus.PUSHING_OUTPUT, + CustomJobStatus.CANCELING, + } + except Exception: + return { + 820, # CustomJobStatus.FETCHING_INPUT + 821, # CustomJobStatus.CONVERTING_INPUT + 822, # CustomJobStatus.CONVERTING_OUTPUT + 823, # CustomJobStatus.PUSHING_OUTPUT + 824, # CustomJobStatus.CANCELING + } + + +@functools.cache +def _jobStateMap(): + """The girder ``JobStatus`` -> neutral projected-state map, built once.""" + + return { + JobStatus.INACTIVE: "pending", + JobStatus.QUEUED: "pending", + JobStatus.RUNNING: "running", + JobStatus.SUCCESS: "success", + JobStatus.ERROR: "error", + JobStatus.CANCELED: "cancelled", + } + + +def isTerminalStatus(status): + """Whether a Girder ``JobStatus`` is terminal (SUCCESS / ERROR / CANCELED). + + The single definition of "the job has settled": the terminal-time scan, the + ownership deletion guard, the transient-input cleanup, and the DELETE route + all read it, so what counts as terminal cannot drift between them. + """ + return status in terminalStatuses() + + +@functools.cache +def terminalStatuses(): + """The terminal ``JobStatus`` set itself (for Mongo ``$nin`` queries).""" + + return frozenset({JobStatus.SUCCESS, JobStatus.ERROR, JobStatus.CANCELED}) + + +def _projectJobState(job): + """The neutral projected job state (a ``jobStateSchema`` value) from Girder's + ``JobStatus``. + + The single shared JobStatus->state map, reached through ``_projectJobFacts`` + (and ``_readableOutputFilesForJobs``), so the status and history reads cannot + disagree about execution state. Neutral names only — never the girder + ``JobStatus`` enum on the wire. + An unknown status maps to ``"pending"`` (fail closed), except girder_worker's + active states, which project to ``"running"`` so an active job never regresses. + Output publication never changes this execution state; ``_projectJobFacts`` + carries result readiness separately. + """ + status = job.get("status") + state = _jobStateMap().get(status) + if state is not None: + return state + if status in _workerActiveStates(): + return "running" + return "pending" + + +def _progressRatio(job): + """The job's clamped ``[0, 1]`` progress ratio, or ``None`` when unavailable. + + Shared by the status projection and the history summary so the summary never + rebuilds the full status projection (errorTail log join included) just to read + progress. The clamp is load-bearing: a worker/CLI reporting >100% (or a + negative) fails the client's ``min(0).max(1)`` history-page schema and makes it + reject the WHOLE page, losing re-discovery for every job in it. + """ + progress = job.get("progress") or {} + if not progress.get("total") or progress.get("current") is None: + return None + try: + ratio = float(progress["current"]) / float(progress["total"]) + except (TypeError, ValueError, ZeroDivisionError): + # ValueError: updateJob stores whatever a writer PUT — a non-numeric + # current/total string must not 500 the whole history page. + return None + return min(1.0, max(0.0, ratio)) + + +def _projectJobStatus(job, user=None): + """Convert Girder Job status to ProcessingJobStatus.""" + facts = _projectJobFacts(job, user) + state = facts["state"] + out = { + "jobId": str(job["_id"]), + "state": state, + "resultState": facts["resultState"], + } + if state == "error": + log = job.get("log") or [] + if isinstance(log, list): + tail = "".join(log[-20:]) + else: + tail = str(log)[-2000:] + out["errorTail"] = tail + progress = _progressRatio(job) + if progress is not None: + out["progress"] = progress + return out + + +def _transitionTime(job, status): + for transition in job.get("timestamps") or []: + if isinstance(transition, dict) and transition.get("status") == status: + return transition.get("time") + return None + + +def _terminalTime(job): + """The job's most recent terminal transition time, or ``None``.""" + for transition in reversed(job.get("timestamps") or []): + if isinstance(transition, dict) and isTerminalStatus(transition.get("status")): + return transition.get("time") + return None + + +def _outputSummary(job, user, facts=None): + """Neutral output-health counts for the job-history wire shape. + + - ``recorded``: declared outputs that recorded AND resolve to a readable file. + - ``missing``: settled declared outputs that never recorded, plus recorded + ids whose file is gone/unreadable. + """ + facts = facts or _projectJobFacts(job, user) + return { + "recorded": len(facts["resolved"]), + "missing": facts["missing"], + } + + +def _projectJobHistorySummary(job, user, readableOutputFiles=None): + """Project one Girder job into the lightweight history wire shape.""" + + creatorName = ( + " ".join( + filter( + None, + [ + user.get("firstName"), + user.get("lastName"), + ], + ) + ) + or user.get("login") + or str(user.get("_id") or "") + ) + facts = _projectJobFacts(job, user, readableOutputFiles=readableOutputFiles) + summary = { + "jobId": str(job["_id"]), + "taskId": str(job.get(_TASK_ID_FIELD) or ""), + "taskTitle": str(job.get("title") or job.get(_TASK_ID_FIELD) or ""), + "createdBy": {"id": str(user["_id"]), "name": creatorName}, + "createdAt": _toIso(job.get("created")) or "", + "state": facts["state"], + "resultState": facts["resultState"], + "outputSummary": _outputSummary(job, user, facts), + } + startedAt = _toIso(_transitionTime(job, JobStatus.RUNNING)) + finishedAt = _toIso(_terminalTime(job)) + if startedAt: + summary["startedAt"] = startedAt + if finishedAt: + summary["finishedAt"] = finishedAt + progress = _progressRatio(job) + if progress is not None: + summary["progress"] = progress + return summary + + +def _intentForOutput(out, url, name, providerId, jobId): + """Build the declarative result intent for one output. + + Results cross the wire as declarative intents the client's single applier + applies — never a ``role`` the client switches on. The vocabulary the client + validates (VolView ``backend-contract/processing/wire.ts``): a labelmap → + ``add-segment-group``, a plain image → ``add-base-image``. Any other file + remains an ordinary result record with no state directive. + + A labelmap intent carries a provider-qualified + ``source: {providerId, jobId, outputId}`` provenance tag (``outputId`` = the + CLI's output identifier) so the idempotency key remains unique when two + providers use the same raw job/output ids and round-trips the + ``.volview.zip``. A labelmap's segment names/colors travel *inside* the + ``.seg.nrrd`` file as embedded metadata and are read client-side, so the + backend sets no ``segments`` payload; the wire field stays optional. + Validates against the contract ``result-intent`` schema. + """ + fileRef = {"url": url, "name": name} + if out["isLabel"]: + return { + "intent": "add-segment-group", + **fileRef, + "source": { + "providerId": str(providerId), + "jobId": str(jobId), + "outputId": out["name"], + }, + } + if out["tag"] == "image": + return {"intent": "add-base-image", **fileRef} + return fileRef + + +def _recordedJobOutputs(job): + """The ``{identifier: fileId}`` map the upload-finalization handler + recorded (or {}).""" + outputs = (job or {}).get(_OUTPUTS_FIELD) + return dict(outputs) if isinstance(outputs, dict) else {} + + +def _recordedOutputSpecs(job): + """The declared output specs recorded at submit (or []).""" + specs = (job or {}).get(_OUTPUT_SPECS_FIELD) + return list(specs) if isinstance(specs, list) else [] + + +def _projectJobFacts(job, user, readableOutputFiles=None): + """Project the canonical execution and output-readiness facts for one job. + + Status, history, and result reads all consume this projection so execution + state never changes to hide output publication, and missing-output accounting + cannot disagree between endpoints. File readability comes from the ONE + batched loader (``_readableOutputFilesForJobs``): the history page passes its + page-wide map in, single-job callers omit it and the loader runs for just this + job — either way the same two-query ACL check decides readability. + """ + state = _projectJobState(job) + if state in {"pending", "running"}: + return { + "state": state, + "resultState": "waiting", + "resolved": [], + "missing": 0, + } + if state in {"error", "cancelled"}: + return { + "state": state, + "resultState": "unavailable", + "resolved": [], + "missing": 0, + } + + specs = { + spec["name"]: spec + for spec in _recordedOutputSpecs(job) + if isinstance(spec, dict) and isinstance(spec.get("name"), str) + } + recorded = _recordedJobOutputs(job) + if readableOutputFiles is None: + readableOutputFiles = _readableOutputFilesForJobs([job], user) + resolved = [] + unreadable = 0 + for outputId, spec in specs.items(): + fileId = recorded.get(outputId) + if fileId is None: + continue + fileDoc = readableOutputFiles.get(str(fileId)) + if fileDoc is None: + unreadable += 1 + else: + resolved.append({"out": spec, "fileDoc": fileDoc}) + + unrecorded = len(set(specs) - set(recorded)) + missing = unrecorded + unreadable + return { + "state": state, + "resultState": "incomplete" if missing else "ready", + "resolved": resolved, + "missing": missing, + } + + +def _readableOutputFilesForJobs(jobs, user): + """Load all readable declared output files for a set of jobs in two queries. + + The ONE loading path deciding output readability — the history page passes + its whole page, and ``_projectJobFacts`` routes single-job status/result + reads through it too, so the ACL semantics cannot drift between endpoints. + The recorded/missing counts are readability-aware, so the ACL check is + load-bearing; ``inputs.readableFilesById`` is the shared batched boundary. + Files with invalid ids, missing parents, or unreadable parents are absent + from the map and therefore count as missing. The returned map is keyed by + string id because persisted output ids and model documents may use different + ObjectId/string representations. The projection carries + ``name``/``mimeType``/``size`` because ``_collectJobResults`` builds result + intents (download url + file metadata) from these same docs. + """ + fileIdsByString = {} + for job in jobs: + if _projectJobState(job) != "success": + continue + declared = _declaredOutputIdentifiers(job) + for outputId, fileId in _recordedJobOutputs(job).items(): + if outputId not in declared or fileId is None: + continue + try: + objectId = fileId if isinstance(fileId, ObjectId) else ObjectId(fileId) + except (TypeError, ValueError): + continue + fileIdsByString.setdefault(str(objectId), objectId) + if not fileIdsByString: + return {} + return readableFilesById( + fileIdsByString.values(), + user, + fields={"_id": 1, "itemId": 1, "name": 1, "mimeType": 1, "size": 1}, + ) + + +def _collectJobResults(job, user, facts=None): + """Resolve a job's outputs to declarative result intents — reference-bound. + + Reads the ``{identifier: fileId}`` map ``outputs._recordJobOutput`` recorded ON + the job (never a folder-name scan, never ``_original_params``), resolves the + files through the batched READ-permission loader, and projects each into its + result intent with a ``makeFileDownloadUrl`` download url — origin-relative and + filename-encoded, so non-default API mounts work. Returns ``(results, + missing)`` where ``missing`` counts settled unrecorded outputs plus recorded + outputs whose file is gone/unreadable — loss is countable, never a silently + shorter list. Two concurrent same-name jobs can never cross results: each reads + only the ids bound to itself. + """ + facts = facts or _projectJobFacts(job, user) + resolved = facts["resolved"] + + # The wire shape is the intent object itself — `{intent, url, name, source?}` + # — plus the `id`/`mimeType`/`size` file metadata the client's JobList reads. + providerId = processingProviderId(job[_LAUNCH_FOLDER_FIELD]) + results = [] + for entry in resolved: + out = entry["out"] + fileDoc = entry["fileDoc"] + url = makeFileDownloadUrl(fileDoc) + intent = _intentForOutput( + out, url, fileDoc["name"], providerId, job["_id"] + ) + result = { + **intent, + "id": str(fileDoc["_id"]), + "mimeType": fileDoc.get("mimeType"), + "size": fileDoc.get("size"), + } + results.append(result) + return results, facts["missing"] + + +def _jobResultsPayload(job, user): + """Apply honest result-read semantics and return the wire result envelope. + + Waiting and unavailable results return the typed conflict body the + route serves as HTTP 409. Ready and incomplete reads return HTTP 200; partial + and total output loss both use an incomplete envelope with an accurate count. + """ + facts = _projectJobFacts(job, user) + state = facts["state"] + resultState = facts["resultState"] + if resultState == "unavailable": + return { + "code": "results_unavailable", + "message": "Job %s results are unavailable (state=%s)" + % (job.get("_id"), state), + "state": state, + "resultState": resultState, + } + if resultState == "waiting": + return { + "code": "results_not_ready", + "message": "Job %s results are not ready (resultState=%s)" + % (job.get("_id"), resultState), + "state": state, + "resultState": resultState, + } + results, missing = _collectJobResults(job, user, facts) + if missing: + logger.info( + "[volview_processing] job %s: %d declared output(s) missing", + job.get("_id"), + missing, + ) + return {"resultState": resultState, "intents": results, "missing": missing} diff --git a/girder_volview/backend/routes.py b/girder_volview/backend/routes.py new file mode 100644 index 0000000..e556532 --- /dev/null +++ b/girder_volview/backend/routes.py @@ -0,0 +1,940 @@ +"""Processing backend -- REST routes + job creation + route registration. + +Cross-module helper calls are MODULE-QUALIFIED on purpose (``submit._foo`` / +``inputs._foo`` / ``outputs._foo`` / ``results._foo``) so a test that patches a +helper on its DEFINING module reaches the call site here; a bare +``from .submit import _foo`` binds a name ``setattr(submit, "_foo", ...)`` +cannot reach. +""" + +import base64 +import binascii +import copy +import datetime +import json +import threading +import time +import uuid + +import cherrypy +from girder import events, logger +from girder.api import access +from girder.api.describe import Description, autoDescribeRoute +from girder.api.rest import Resource, boundHandler +from girder.constants import AccessType, SortDir, TokenScope +from girder.exceptions import RestException, ValidationException +from girder.models.folder import Folder + +# Module-object import (not ``from ... import Job``): call sites resolve +# ``girder_job.Job`` at call time, so tests may monkeypatch the class on +# ``girder_jobs.models.job`` and be seen here. +from girder_jobs.models import job as girder_job + +from ..utils import ( + _toIso, + makeFileDownloadUrl, + JOB_OUTPUT_FOLDER_META_KEY, + TRANSIENT_STAGED_META_KEY, +) +from .config import PROCESSING_ROUTE_NAME +from .slicer_spec import declared_params, translate_slicer_xml, validate_task_spec +from . import inputs, submit, outputs, results + + +@access.public(cookie=True, scope=TokenScope.DATA_READ) +@boundHandler +@autoDescribeRoute( + Description("List processing tasks available for a folder.") + .modelParam("folderId", model=Folder, level=AccessType.READ) + .produces(["application/json"]) +) +def listTasks(self, folder): + user = self.getCurrentUser() + tasks = [] + if user and submit._slicerCliAvailable(): + try: + tasks.extend( + [submit._cliItemToSummary(c) for c in submit._scopedCliItems(user)] + ) + except Exception: + logger.exception("Failed to list slicer_cli_web items") + return tasks + + +# Explicit short lifetime for the CLI-container token. Without ``days`` Girder +# applies ``core.cookie_lifetime`` (180 days by default), leaving a broad +# data-plane credential valid for months after the job ends. One day covers queue +# wait + run time and bounds exposure if the token leaks (a compromised image, +# broker, or command-line capture). +_CONTAINER_TOKEN_TTL_DAYS = 1.0 + +JOB_HISTORY_PAGE_DEFAULT = 25 +JOB_HISTORY_PAGE_MAX = 100 +JOB_HISTORY_INDEX = "volview_job_history" +JOB_OUTPUT_FOLDER_INDEX = "volview_output_folder" +_SUBMISSION_ID_FIELD = "volviewSubmissionId" +_SUBMITTED_PARAMETERS_FIELD = "volviewSubmittedParameters" + + +def ensureJobHistoryIndexes(jobModel=None): + """Install the indexes the history list and output-correlation queries need. + + A compound index backs the personal newest-first history page. A point-lookup + index on the private output-folder id backs ``outputs._jobForOutputFolder``, + which runs a ``findOne`` for EVERY finalized output upload -- without it each + correlation is a full jobs-collection scan that worsens as history grows. + """ + if jobModel is None: + jobModel = girder_job.Job() + jobModel.collection.create_index( + [ + (inputs._LAUNCH_FOLDER_FIELD, 1), + ("userId", 1), + ("created", -1), + ("_id", -1), + ], + name=JOB_HISTORY_INDEX, + ) + jobModel.collection.create_index( + [(outputs._OUTPUT_FOLDER_ID_FIELD, 1)], + name=JOB_OUTPUT_FOLDER_INDEX, + ) + + +def _encodeJobCursor(job): + payload = json.dumps( + { + "created": _toIso(job.get("created")), + "id": str(job["_id"]), + }, + separators=(",", ":"), + ).encode("utf8") + return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=") + + +def _decodeJobCursor(cursor): + from bson.objectid import ObjectId + + try: + padded = cursor + "=" * (-len(cursor) % 4) + value = json.loads(base64.urlsafe_b64decode(padded).decode("utf8")) + created = datetime.datetime.fromisoformat(value["created"]) + if created.tzinfo is not None: + created = created.astimezone(datetime.timezone.utc).replace(tzinfo=None) + return created, ObjectId(value["id"]) + except (TypeError, ValueError, KeyError, json.JSONDecodeError, binascii.Error): + raise RestException("Invalid job history cursor", code=400) from None + + +def _jobHistoryPageSize(limit): + try: + pageSize = int(limit if limit is not None else JOB_HISTORY_PAGE_DEFAULT) + if pageSize < 1 or pageSize > JOB_HISTORY_PAGE_MAX: + raise ValueError() + return pageSize + except (TypeError, ValueError): + raise RestException("Invalid job history limit", code=400) from None + + +def _jobCursorContinuation(cursor): + created, jobId = _decodeJobCursor(cursor) + return [ + {"created": {"$lt": created}}, + {"created": created, "_id": {"$lt": jobId}}, + ] + + +@access.public(cookie=True, scope=TokenScope.DATA_READ) +@boundHandler +@autoDescribeRoute( + Description("List the current user's complete processing-job history.") + .notes( + "Returns a bounded newest-first page of lightweight summaries. The " + "continuation cursor is opaque; pagination bounds responses, never " + "history retention. Logs and submitted parameters are detail-only." + ) + .modelParam("folderId", model=Folder, level=AccessType.READ) + .param("limit", "Page size (1-100).", required=False, dataType="integer") + .param("cursor", "Opaque continuation cursor.", required=False) + .produces(["application/json"]) +) +def listJobHistory(self, folder, limit=JOB_HISTORY_PAGE_DEFAULT, cursor=None): + user = self.getCurrentUser() + if not user: + return {"jobs": [], "nextCursor": None} + + pageSize = _jobHistoryPageSize(limit) + query = { + inputs._LAUNCH_FOLDER_FIELD: str(folder["_id"]), + "userId": user["_id"], + } + if cursor: + query["$or"] = _jobCursorContinuation(cursor) + found = girder_job.Job().findWithPermissions( + query=query, + user=user, + jobUser=user, + level=AccessType.READ, + sort=[("created", SortDir.DESCENDING), ("_id", SortDir.DESCENDING)], + limit=pageSize + 1, + # The summary projection never reads the log, which is unbounded (multi-MB + # on chatty/failed CLIs), so exclude it from every page. Mirrors + # Job.load(includeLog=False)'s {'log': False} projection. + fields={"log": False}, + ) + page = list(found) + hasMore = len(page) > pageSize + page = page[:pageSize] + readableOutputFiles = results._readableOutputFilesForJobs(page, user) + return { + "jobs": [ + results._projectJobHistorySummary( + job, user, readableOutputFiles=readableOutputFiles + ) + for job in page + ], + "nextCursor": _encodeJobCursor(page[-1]) if hasMore else None, + } + + +@access.public(cookie=True, scope=TokenScope.DATA_READ) +@boundHandler +@autoDescribeRoute( + Description("Get the VolView task spec for a task.") + .modelParam("folderId", model=Folder, level=AccessType.READ) + .param("taskId", "The task identifier.", paramType="path") +) +def getTaskSpec(self, folder, taskId): + # The Slicer XML is translated into VolView's task spec server-side, so the + # client never parses backend XML. + user = self.getCurrentUser() + if not submit._slicerCliAvailable(): + raise RestException("slicer_cli_web is not installed", code=404) + scoped = submit._findScopedCliItem(taskId, user) + if not scoped: + raise RestException("Unknown taskId", code=404) + # translate_slicer_xml needs the strict parse (title/description + # + ordered params), which ``parse_cli`` does not carry, so it parses the XML + # itself; the scoped parse is consumed by runTask. + cliItem, _parsedCli = scoped + try: + return validate_task_spec(translate_slicer_xml(cliItem.xml, str(taskId))) + except ValueError as exc: + logger.error("Invalid VolView task spec for task %s: %s", taskId, exc) + raise RestException("Task specification is invalid", code=500) from None + + +# The single server-owned container every per-job output folder nests inside: one +# hierarchy entry per launch folder no matter how many jobs accumulate, and one +# ADMIN-gated "clear this dataset's job history" gesture (removing it recurses +# through the per-job folders, firing the reverse cascade per job). The name is +# reserved: a pre-existing USER folder with this name is never adopted -- reuse is +# gated on the server-owned marker, and an unmarked name collision refuses the +# submission (409) instead. +JOBS_CONTAINER_NAME = "volview-jobs" + + +def _jobsContainerFolder(launchFolder, user): + """Create-or-reuse the launch folder's server-owned ``volview-jobs`` container. + + Reuse requires the ``volviewJobOutputFolder`` marker, the server-owned + identity stamped at creation -- the reserved name alone is not enough: + + * Why a marker: adopting a user's pre-existing folder that merely shares + the reserved name would silently hide its contents from launch + manifests and turn the container-delete gesture into "delete unrelated + user data". An unmarked name collision refuses the submission with a + 409 instead of being adopted. + * Why the marker can be trusted: it is stamped ONLY on a folder this call + itself created. ``createFolder`` never reuses an existing folder (a + name collision raises ``ValidationException``), so a folder someone + else made in the check-create window re-runs the marker check instead + of being adopted. + * Grace period: create and stamp are two separate writes, so an unmarked + collision gets a short grace period -- room for a concurrent + submission's own container to land between its create and its stamp -- + before the 409 fires. + * Failed stamp: if the stamp write fails, the just-created folder is + removed rather than left behind unmarked, which would 409 every future + submission. + * ACL: after creation, the container's ACL is replaced with the launch + folder's exact user and group policy. This removes the implicit ADMIN + grant ``createFolder`` gives its creator while retaining collaborators' + inherited access. Each per-job folder inside keeps its own + submitter-only ACL. + """ + + def findContainer(): + return Folder().findOne( + { + "parentId": launchFolder["_id"], + "parentCollection": "folder", + "name": JOBS_CONTAINER_NAME, + } + ) + + def isMarked(folderDoc): + return bool((folderDoc.get("meta") or {}).get(JOB_OUTPUT_FOLDER_META_KEY)) + + def awaitMarkedContainer(): + """The existing marked container, ``None`` when absent, or a 409 for an + unmarked collision that outlasts the create->stamp grace period.""" + for _ in range(10): + existing = findContainer() + if existing is None or isMarked(existing): + return existing + time.sleep(0.05) + raise RestException( + "A folder named '%s' already exists here and is not a processing " + "jobs container; rename or remove it to run processing tasks" + % JOBS_CONTAINER_NAME, + code=409, + ) + + container = awaitMarkedContainer() + if container is not None: + return container + try: + created = Folder().createFolder( + parent=launchFolder, + name=JOBS_CONTAINER_NAME, + parentType="folder", + creator=user, + public=False, + ) + except ValidationException: + # Lost a creation race: a same-named sibling appeared between the check + # and the create. Re-run the marker check -- a concurrent submission's + # container is reused; a user's folder 409s (never adopted). + container = awaitMarkedContainer() + if container is None: + raise + return container + try: + created = Folder().setAccessList( + created, + launchFolder.get("access", {"users": [], "groups": []}), + save=True, + force=True, + ) + return Folder().setMetadata(created, {JOB_OUTPUT_FOLDER_META_KEY: True}) + except Exception: + Folder().remove(created) + raise + + +def _createJobOutputFolder(launchFolder, user, submissionId): + """Create the job's private, server-owned output folder for a submission. + + Lives inside the launch folder's ``volview-jobs`` container. Every + declared output is forced into this folder, and it is the SOLE + output-correlation + ownership key. Two steps make it private: + + * mark it ``volviewJobOutputFolder`` so the launch manifest excludes it and + its contents (job results take the job path only, never ordinary launch + data); + * REPLACE its ACL with a submitter-only ADMIN list. ``createFolder`` copies + the parent's ACL (``copyAccessPolicies``), which would otherwise leave + every launch-folder collaborator able to read the private results; + ``setAccessList(..., force=True, setPublic=False)`` strips that. Girder + system administrators keep their normal force access. + """ + created = Folder().createFolder( + parent=_jobsContainerFolder(launchFolder, user), + name="volview-job-%s" % submissionId, + parentType="folder", + creator=user, + public=False, + reuseExisting=False, + ) + try: + folder = Folder().setMetadata(created, {JOB_OUTPUT_FOLDER_META_KEY: True}) + Folder().setAccessList( + folder, + { + "users": [{"id": user["_id"], "level": AccessType.ADMIN}], + "groups": [], + }, + save=True, + force=True, + setPublic=False, + ) + return folder + except Exception: + try: + Folder().remove(created) + except Exception: + logger.exception( + "Failed to remove partially initialized job output folder %s", + created.get("_id"), + ) + raise + + +def _removeJobOutputFolder(folder): + """Best-effort removal of a pre-publication output folder (no job yet). + + Only called when a submission failed BEFORE any job was created, so no + ownership record exists to drive the normal deletion cascade. An + already-missing folder is a no-op. + """ + if not folder: + return + try: + Folder().remove(folder) + except Exception: + logger.exception( + "Failed to remove orphaned job output folder %s", folder.get("_id") + ) + + +def _requestCliItem(cliItem, initialJobFields): + """Copy a catalog CLI item and inject request-local initial job fields.""" + requestItem = copy.copy(cliItem) + requestItem.item = copy.deepcopy(getattr(cliItem, "item", {}) or {}) + meta = requestItem.item.setdefault("meta", {}) + dockerParams = meta.setdefault("docker-params", {}) + if not isinstance(dockerParams, dict): + raise ValidationException("CLI docker parameters are malformed") + catalogFields = dockerParams.get("girder_job_other_fields") or {} + if not isinstance(catalogFields, dict): + raise ValidationException("CLI initial job fields are malformed") + merged = dict(catalogFields) + merged.update(initialJobFields) + dockerParams["girder_job_other_fields"] = merged + return requestItem + + +def _genDockerJob(cliItem, params, user, initialJobFields): + """Create the slicer_cli_web docker job for a CLI item and return its doc. + + Isolated as the single live slicer_cli_web touch point so ``runTask`` (and + its tests) can drive job creation without the optional dependency. + """ + from girder.models.token import Token + from slicer_cli_web.rest_slicer_cli import genHandlerToRunDockerCLI + + # Scope-limit the container token to the data plane the CLI actually needs: + # read its inputs, write its outputs. Narrower than the ecosystem norm + # (slicer_cli_web mints full-auth tokens) without weakening Girder ACLs — the + # submitter's own read/write reach still bounds it. It is NOT persisted on the + # job or used as an ownership/correlation key (outputs bind by their private + # parent folder, never by a token). + token = Token().createToken( + user=user, + scope=[TokenScope.DATA_READ, TokenScope.DATA_WRITE], + days=_CONTAINER_TOKEN_TTL_DAYS, + ) + requestItem = _requestCliItem(cliItem, initialJobFields) + handler = genHandlerToRunDockerCLI(requestItem) + # slicer_cli_web only substitutes its GirderApiUrl()/GirderToken() runtime + # transforms when these keys are present in the params it processes + # (prepare_task `_add_optional_input_param` skips a param absent from args). + # The REST route defaults them in, but this backend calls `subHandler` + # directly. Empty -> the transforms substitute, and GirderToken resolves to + # THIS scoped token, not a broader one. Without them a CLI that fetches its own + # inputs by id (`reference="_girder_id_"`, e.g. a multi-file DICOM series) has + # no way to reach Girder. Harmless for a CLI that declares neither -- + # slicer_cli_web ignores undeclared args. + params = dict(params) + params.setdefault("girderApiUrl", "") + params.setdefault("girderToken", "") + # Take a copy so the handler can mutate freely. + job_obj = handler.subHandler(requestItem, copy.deepcopy(params), user, token) + job = job_obj.job if hasattr(job_obj, "job") else job_obj + return job + + +def _prepareSubmissionFields( + submissionId, folder, taskId, values, outputSpecs, transientItemIds, outputFolder +): + """Build every job association field before task publication. + + Includes the job's private output-folder id (``_OUTPUT_FOLDER_ID_FIELD``) — + the sole output-correlation + ownership key — so the folder id is part of the + FIRST job insert, queryable before any worker upload can race in. + + ``outputSpecs`` is the ``slicer_spec.parse_cli`` output descriptor list + ``runTask`` already parsed, threaded in rather than re-parsed here. + """ + fields = { + _SUBMISSION_ID_FIELD: submissionId, + inputs._LAUNCH_FOLDER_FIELD: str(folder["_id"]), + inputs._TASK_ID_FIELD: str(taskId), + outputs._OUTPUT_SPECS_FIELD: outputSpecs, + outputs._OUTPUT_FOLDER_ID_FIELD: str(outputFolder["_id"]), + outputs._OUTPUTS_FIELD: {}, + _SUBMITTED_PARAMETERS_FIELD: copy.deepcopy(values), + } + if transientItemIds: + fields[TRANSIENT_STAGED_META_KEY] = list(transientItemIds) + return fields + + +def _jobForSubmission(submissionId): + + return girder_job.Job().findOne({_SUBMISSION_ID_FIELD: submissionId}) + + +@access.public(cookie=True, scope=TokenScope.DATA_WRITE) +@boundHandler +@autoDescribeRoute( + Description("Submit a processing task.") + .modelParam("folderId", model=Folder, level=AccessType.WRITE) + .param("taskId", "The task identifier.", paramType="path") + .jsonParam( + "body", + "Submission payload: { values: { paramName: ProcessingValue, ... } }", + paramType="body", + required=False, + ) +) +def runTask(self, folder, taskId, body): + user = self.getCurrentUser() + values = (body or {}).get("values", {}) if isinstance(body, dict) else {} + if not isinstance(values, dict): + raise RestException("values must be an object of parameter values", code=400) + + # Reject a payload carrying reserved credentials before any task lookup or + # work; declaration-aware screens run after the CLI XML is parsed below. + submit._rejectReservedSubmitParams(values) + + if not submit._slicerCliAvailable(): + raise RestException("slicer_cli_web is not installed", code=500) + + scoped = submit._findScopedCliItem(taskId, user) + if not scoped: + raise RestException("Unknown taskId", code=404) + cliItem, parsedCli = scoped + + # The submission parses the CLI XML exactly twice: ``_findScopedCliItem`` + # already ran ``parse_cli``, whose ``outputs`` are reused here, and + # ``declared_params`` supplies the label-independent key/value declaration the + # grouped walk can't. Downstream guard/translate steps read these structures. + declared = declared_params(cliItem.xml) + outputSpecs = parsedCli["outputs"] + + # Screens the RAW client keys, so it must run before autofill adds + # server-owned output structures. A synthesized-folder collision, an + # undeclared key, an out-of-declaration value, or a missing required input + # is a boundary 400 naming the parameter, not a later job failure. + submit._rejectSynthesizedFolderParams(values, declared) + submit._rejectUndeclaredSubmitParams(values, declared) + submit._validateDeclaredSubmitValues(values, declared) + submit._rejectMissingRequiredParams(values, declared) + + # Auto-generate a deterministic output filename for any output param the user + # didn't fill (input file + CLI name + parameter name + extension). Names need + # not be unique: outputs bind to the job by its private output folder, not by + # name, so a duplicate filename can never cross results. + values = submit._autofillOutputs(dict(values), outputSpecs, cliItem.name) + + # The output folder is created BEFORE translating params or publishing the + # task: every declared output is forced into it, and its id is part of the + # first job insert so it is queryable before any worker upload can race in. + submissionId = uuid.uuid4().hex + outputFolder = _createJobOutputFolder(folder, user, submissionId) + + transientItemIds = [] + try: + # Resolves each bound input once (own-scheme validation + per-user ACL + # re-check), forces every declared output into the private output folder, + # and rejects a client-supplied folderRef. The authorized file documents + # are reused for transient detection so each URI's ACL check runs once. + params, resolvedInputFiles = submit._translateValuesToSlicerParams( + values, user, outputFolder, declared + ) + # Per-job input ownership: any staged (transient) input is COPIED into the + # job's private folder and the CLI params are rewritten onto the copies. + # The copies are recorded on the job so + # inputs._cleanupTransientOnJobDone deletes them at terminal state; the + # shared staged original is never a job dependency. + params, transientItemIds = inputs.copyStagedInputsIntoJobFolder( + params, resolvedInputFiles, user, outputFolder + ) + # INFO carries only routing identity; the translated CLI params can hold + # sensitive string values, so they stay at debug. + logger.info( + "[volview_processing] runTask folder=%s task=%s submission=%s", + folder["_id"], + taskId, + submissionId, + ) + logger.debug("[volview_processing] runTask params=%s", params) + + initialFields = _prepareSubmissionFields( + submissionId, + folder, + taskId, + values, + outputSpecs, + transientItemIds, + outputFolder, + ) + job_doc = _genDockerJob(cliItem, params, user, initialFields) + except Exception: + # run.delay can fail after Girder Worker's before_task_publish handler + # inserted the job. Resolve that ambiguity by the server-minted id. + job_doc = _jobForSubmission(submissionId) + if job_doc is None: + # No job exists (including a folderRef-rejection 400 before any work): + # remove the pre-publication output folder and staged inputs. + _removeJobOutputFolder(outputFolder) + inputs._removeTransientItems(transientItemIds) + else: + # A job WAS created: cancel it but RETAIN its ownership record so the + # normal terminal + deletion cascade cleans the output folder safely. + try: + + girder_job.Job().cancelJob(job_doc) + except Exception: + logger.exception( + "Failed to cancel ambiguously published job %s", + job_doc.get("_id"), + ) + raise + return {"jobId": str(job_doc["_id"])} + + +# Job-addressed routes are keyed by job id alone and gated by the job's OWN ACL. +# The launch folder is not part of a job's identity, so these carry no +# ``folderId``; they live on the folder-free ``volview_processing`` resource +# below. getJob / getJobResults are READ-gated; cancel and delete are WRITE-gated +# so a read-only viewer who can see a job's status cannot cancel or delete it. + + +def _loadJobForStatusProjection(jobId, user): + """Load a job for status projection, WITHOUT its log unless it is needed. + + The client polls status every ~2s per live job and the job log grows + unbounded, so the common load excludes it at the Mongo projection level + (``includeLog`` defaults False). Only the terminal-error projection reads it + (a bounded tail in ``results._projectJobStatus``), and the poller stops at + terminal, so the log is reloaded at most once per job. The detail route is + the full-log path. + """ + + job = girder_job.Job().load(jobId, user=user, level=AccessType.READ, exc=True) + if results._projectJobState(job) == "error": + job = girder_job.Job().load( + jobId, + user=user, + level=AccessType.READ, + exc=True, + includeLog=True, + ) + return job + + +@access.public(cookie=True, scope=TokenScope.DATA_READ) +@boundHandler +@autoDescribeRoute( + Description("Get job status.") + .param("jobId", "The job identifier.", paramType="path") + .produces(["application/json"]) +) +def getJob(self, jobId): + user = self.getCurrentUser() + job = _loadJobForStatusProjection(jobId, user) + return results._projectJobStatus(job, user) + + +@access.public(cookie=True, scope=TokenScope.DATA_READ) +@boundHandler +@autoDescribeRoute( + Description("Get detail-only job logs and submitted parameters.") + .param("jobId", "The job identifier.", paramType="path") + .produces(["application/json"]) +) +def getJobHistoryDetail(self, jobId): + user = self.getCurrentUser() + + job = girder_job.Job().load( + jobId, + user=user, + level=AccessType.READ, + exc=True, + includeLog=True, + ) + log = job.get("log") or [] + if not isinstance(log, list): + log = [str(log)] + parameters = job.get(_SUBMITTED_PARAMETERS_FIELD) or {} + if not isinstance(parameters, dict): + parameters = {} + return { + "jobId": str(job["_id"]), + "log": [str(line) for line in log], + "parameters": parameters, + } + + +@access.public(cookie=True, scope=TokenScope.DATA_WRITE) +@boundHandler +@autoDescribeRoute( + Description( + "Delete a terminal job and its owned output folder + staged inputs." + ) + .notes( + "A pending or running job returns 409 (cancel it first). A terminal job " + "is removed together with its private output folder and any remaining " + "staged inputs — deleting the job also deletes its results." + ) + .param("jobId", "The job identifier.", paramType="path") + .errorResponse("Write access was denied for the job.", 403) + .errorResponse("The job is still running.", 409) +) +def deleteJob(self, jobId): + user = self.getCurrentUser() + + model = girder_job.Job() + job = model.load(jobId, user=user, level=AccessType.WRITE, exc=True) + # The model.job.remove handler ALSO enforces this (protecting other + # Job.remove callers); the route returns the typed product response + # rather than the model's raise. + if not results.isTerminalStatus(job.get("status")): + raise RestException( + "Job is still running; cancel it before deleting", code=409 + ) + # The ownership cascade (owned output folder + staged inputs) lives in the + # model.job.remove handler; this route must not add a second cascade. + model.remove(job) + cherrypy.response.status = 204 + return None + + +@access.public(cookie=True, scope=TokenScope.DATA_READ) +@boundHandler +@autoDescribeRoute( + Description("Get job results.") + .param("jobId", "The job identifier.", paramType="path") + .produces(["application/json"]) +) +def getJobResults(self, jobId): + user = self.getCurrentUser() + + job = girder_job.Job().load(jobId, user=user, level=AccessType.READ, exc=True) + payload = results._jobResultsPayload(job, user) + if payload.get("code"): + cherrypy.response.status = 409 + if payload["resultState"] == "waiting": + cherrypy.response.headers["Retry-After"] = "2" + return payload + + +@access.public(cookie=True, scope=TokenScope.DATA_WRITE) +@boundHandler +@autoDescribeRoute( + Description("Cancel a job.") + .notes( + "Best-effort: Girder only transitions an INACTIVE/QUEUED/RUNNING job to " + "CANCELED, so cancelling an already-terminal job is a no-op. The response " + "is the job's real projected status after the attempt -- never a " + "fabricated 'cancelled' -- and the client's poller converges on whatever " + "terminal state Girder ultimately reports." + ) + .param("jobId", "The job identifier.", paramType="path") + .produces(["application/json"]) + .errorResponse("Write access was denied for the job.", 403) +) +def cancelJob(self, jobId): + user = self.getCurrentUser() + + jobModel = girder_job.Job() + job = jobModel.load(jobId, user=user, level=AccessType.WRITE, exc=True) + try: + jobModel.cancelJob(job) + except ValidationException: + # Girder refuses a CANCELED transition from a terminal state, so an + # already-finished job cannot be cancelled. Fall through and report the + # job's real state rather than fabricate a `cancelled` the poller would + # contradict. + pass + fresh = _loadJobForStatusProjection(jobId, user) + return results._projectJobStatus(fresh, user) + + +@access.public(cookie=True, scope=TokenScope.DATA_WRITE) +@boundHandler +@autoDescribeRoute( + Description("Stage a parent-bound labelmap as a transient processing input.") + .notes( + "Accepts multipart labelmap bytes plus a neutral reference-image InputValue. " + "The backend validates and resolves that opaque relationship against durable " + "reference files before minting the staged URI. The created item is tagged " + "transient, deleted when its job reaches a terminal state, or swept if " + "never submitted." + ) + .modelParam("folderId", model=Folder, level=AccessType.WRITE) + .param("file", "The labelmap bytes.", paramType="formData", dataType="file") + .jsonParam( + "descriptor", + "Typed labelmap resource descriptor.", + paramType="formData", + requireObject=True, + ) + .errorResponse() +) +def stageInput(self, folder, file, descriptor): + user = self.getCurrentUser() + # Validate the whole descriptor (reference image included) before writing + # bytes so a malformed, foreign, unauthorized, or transient reference never + # leaves an orphan upload. + name = inputs.validateStagedDescriptor(descriptor, user) + # Keep browser-generated working data out of the launch folder. The shared, + # server-owned jobs container is already excluded from VolView's launch + # manifest and carries the launch folder's collaborator ACL. Submission will + # still copy this original into its private per-job folder before publishing + # the task. + stagingFolder = _jobsContainerFolder(folder, user) + # Job-end cleanup never sees an upload that was never submitted, so age out + # this jobs container's staged orphans before adding another. + inputs._sweepOrphanTransients(stagingFolder) + fileDoc = inputs._streamMultipartFileIntoItem(stagingFolder, user, file, name) + try: + inputs._tagItemTransient(fileDoc) + except Exception: + # An untagged item is invisible to both the TTL sweep and the + # launch-manifest exclusion (each keys on the transient marker), so it + # would linger as apparent durable launch data. + inputs._removeTransientItems([fileDoc["itemId"]]) + raise + # The backend mints the staged URI; the client constructs none. + return {"uris": [makeFileDownloadUrl(fileDoc)]} + + +class _JobResource(Resource): + """Folder-free REST surface for the job-addressed routes. + + Mounted at ``/volview_processing`` (a sibling of ``/folder``, ``/item``), + this hosts status / results / cancel keyed by job id alone -- the launch + folder is not part of a job's identity. The launch-context routes + (tasks / spec / run / stage) stay on the folder tree because they operate + per-folder. The handlers are the same module-level ``@boundHandler`` + functions; only their mount point differs. + """ + + def __init__(self): + super().__init__() + self.resourceName = PROCESSING_ROUTE_NAME + self.route("GET", ("jobs", ":jobId"), getJob) + self.route("GET", ("jobs", ":jobId", "detail"), getJobHistoryDetail) + self.route("DELETE", ("jobs", ":jobId"), deleteJob) + self.route("GET", ("jobs", ":jobId", "results"), getJobResults) + self.route("POST", ("jobs", ":jobId", "cancel"), cancelJob) + + +def _ensureJobHistoryIndexesInBackground(): + """Kick the index builds off a daemon thread so plugin load never blocks. + + ``create_index`` is idempotent and a no-op on steady state, but the FIRST + boot against a large pre-existing jobs collection waits for the whole build; + the queries the indexes back merely degrade to scans until the build lands. + """ + + def build(): + try: + ensureJobHistoryIndexes() + except Exception: + logger.exception("Failed to ensure volview job-history indexes") + + threading.Thread( + target=build, name="volview-job-history-indexes", daemon=True + ).start() + + +def addBackendRoutes(info): + _ensureJobHistoryIndexesInBackground() + # Delete a job's transient staged inputs once it reaches a terminal state. + # Fires for every job update but no-ops cheaply unless the job carries the + # transient marker. + events.bind( + "jobs.job.update.after", + "girder_volview.backend.routes", + inputs._cleanupTransientOnJobDone, + ) + # Record each finalized output file's id onto the job that OWNS the file's + # private parent folder, keyed by output identifier, so result collection + # reads ids OFF the job. Fires for every upload but returns early unless the + # upload lands in a job's output folder under a declared identifier. + events.bind( + "model.file.finalizeUpload.after", + "girder_volview.backend.outputs", + outputs._recordJobOutput, + ) + # Ownership cascade: each job owns one private output folder + its staged + # inputs. Running before the DB delete, this refuses to remove a nonterminal + # owned job and cascade-deletes its owned resources, so the DELETE route, + # Girder's built-in job route, and any direct Job.remove caller all + # honor the same terminal guard and cleanup. + events.bind( + "model.job.remove", + "girder_volview.backend.outputs", + outputs._cascadeDeleteJobOwnedResources, + ) + # Reverse ownership cascade: deleting a job's output folder in the Girder + # hierarchy deletes the job record too (refusing for a live job), so folder + # deletion is a first-class "delete this job" gesture and no orphaned job + # rows accumulate. Removing the volview-jobs container recurses per job + # folder. + events.bind( + "model.folder.remove", + "girder_volview.backend.outputs", + outputs._cascadeDeleteFolderOwnedJob, + ) + # REST pre-guard for the same invariant: Folder.remove cleans contents + # BEFORE model.folder.remove fires, so refuse a live job's folder (or a + # container holding one) before the delete handler touches anything. + events.bind( + "rest.delete.folder/:id.before", + "girder_volview.backend.outputs", + outputs._refuseLiveJobFolderRestDelete, + ) + # The same preflight for every OTHER recursive deletion entry point: + # collection delete, user delete, and the batch /resource route all reach + # Folder.remove without passing DELETE /folder/:id, so each would otherwise + # destroy a live job's staged inputs and partial outputs before the late + # model-level guard could refuse. + events.bind( + "rest.delete.collection/:id.before", + "girder_volview.backend.outputs", + outputs._refuseLiveJobCollectionRestDelete, + ) + events.bind( + "rest.delete.user/:id.before", + "girder_volview.backend.outputs", + outputs._refuseLiveJobUserRestDelete, + ) + events.bind( + "rest.delete.resource.before", + "girder_volview.backend.outputs", + outputs._refuseLiveJobResourceRestDelete, + ) + # The recorded id map is READ-exposed; the job's own ACL is the gate + # (otherFields + exposeFields, mirroring slicer_cli_web's slicerCLIBindings). + + girder_job.Job().exposeFields( + level=AccessType.READ, fields={outputs._OUTPUTS_FIELD} + ) + info["apiRoot"].folder.route( + "GET", (":folderId", PROCESSING_ROUTE_NAME, "tasks"), listTasks + ) + # Job re-discovery is context-scoped (it takes a launch folder), not + # job-addressed: a reloaded client GETs this to re-find its jobs. + info["apiRoot"].folder.route( + "GET", (":folderId", PROCESSING_ROUTE_NAME, "jobs"), listJobHistory + ) + info["apiRoot"].folder.route( + "POST", (":folderId", PROCESSING_ROUTE_NAME, "stage"), stageInput + ) + info["apiRoot"].folder.route( + "GET", + (":folderId", PROCESSING_ROUTE_NAME, "tasks", ":taskId", "spec"), + getTaskSpec, + ) + info["apiRoot"].folder.route( + "POST", + (":folderId", PROCESSING_ROUTE_NAME, "tasks", ":taskId", "run"), + runTask, + ) + info["apiRoot"].volview_processing = _JobResource() diff --git a/girder_volview/backend/slicer_spec.py b/girder_volview/backend/slicer_spec.py new file mode 100644 index 0000000..267d31b --- /dev/null +++ b/girder_volview/backend/slicer_spec.py @@ -0,0 +1,816 @@ +"""Slicer Execution Model XML -> VolView task spec. + +The server emits VolView's ``zod``-defined task spec so the client never parses +a backend's XML. VolView's ``backend-contract`` ``task-spec`` golden fixtures +pin the output exactly. + +Pure standard library (``xml.etree``) so it imports without Girder. +""" + +import math +import re +import xml.etree.ElementTree as ET + + +def _first_child(el, tag): + return next((c for c in el if c.tag == tag), None) + + +def _all_children(el, tag): + return [c for c in el if c.tag == tag] + + +def _child_text(el, tag): + child = _first_child(el, tag) + if child is None or child.text is None: + return "" + return child.text + + +# Mapping tables. DO NOT redesign these: the fixtures pin them byte for byte. + +# Slicer element tag -> widget type. An unmapped tag yields ``None`` (the caller +# treats it as an unknown field kind, fail closed). +_TYPE_MAP = { + "integer": "number", + "float": "number", + "double": "number", + "boolean": "boolean", + "string": "string", + "integer-vector": "number-vector", + "float-vector": "number-vector", + "double-vector": "number-vector", + "string-vector": "string-vector", + "integer-enumeration": "number-enumeration", + "float-enumeration": "number-enumeration", + "double-enumeration": "number-enumeration", + "string-enumeration": "string-enumeration", + "region": "region", + "image": "image", + "file": "file", + "item": "item", + "directory": "directory", + "multi": "multi", +} + + +def _widget_type(tag): + return _TYPE_MAP.get(tag) + + +# Leading numeric run, JS ``parseFloat``-style (optional sign, digits, decimal, +# exponent). +_LEADING_FLOAT = re.compile(r"[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?") + + +def _parse_float(value): + # Emulate JS ``parseFloat``: a lenient value (``"1,000"``, ``"50%"``, + # ``"1.5x"``) degrades to its leading number rather than raising + # ``ValueError`` and 500-ing the whole task-spec endpoint. No leading number + # yields NaN. + try: + return float(value) + except (TypeError, ValueError): + match = _LEADING_FLOAT.match(str(value).strip()) + return float(match.group(0)) if match else float("nan") + + +def _convert(widget_type, value): + """Coerce a raw XML string to the widget's value type.""" + if widget_type in ("number", "number-enumeration"): + return _parse_float(value) + if widget_type == "boolean": + return value.lower() == "true" + if widget_type == "number-vector": + return [_parse_float(s) for s in value.split(",")] + if widget_type == "string-vector": + return value.split(",") + return value + + +def _parse_constraints(widget_type, constraints_el): + """```` -> ``{min,max,step}`` (converted).""" + if constraints_el is None: + return {} + spec = {} + minimum = _child_text(constraints_el, "minimum") + maximum = _child_text(constraints_el, "maximum") + step = _child_text(constraints_el, "step") + if minimum: + spec["min"] = _convert(widget_type, minimum) + if maximum: + spec["max"] = _convert(widget_type, maximum) + if step: + spec["step"] = _convert(widget_type, step) + return spec + + +def _parse_default(widget_type, default_el): + """```` -> the converted value, or ``None``. + + Template placeholders (``{{x}}``) are skipped. + """ + if default_el is None: + return None + text = default_el.text or "" + if len(text) == 0: + return None + is_template = text[:2] == "{{" and text[-2:] == "}}" + if is_template: + return None + return _convert(widget_type, text) + + +def _parse_param(param_el, section): + tag = param_el.tag + widget = _widget_type(tag) + channel = "output" if _child_text(param_el, "channel") == "output" else "input" + # ctk_cli identifies a -less param by its longflag with leading dashes + # stripped; slicer_cli_web binds submitted args by that identifier, so the + # id must match or the submitted value is silently ignored. + param_id = ( + _child_text(param_el, "name") or _child_text(param_el, "longflag").lstrip("-") + ).strip() + required = len(_child_text(param_el, "index")) > 0 + values = None + if widget in ("string-enumeration", "number-enumeration"): + values = [ + _convert(widget, el.text or "") for el in _all_children(param_el, "element") + ] + return { + "tag": tag, # the raw Slicer element name + "widget": widget, # WidgetType, or None for an unmapped tag + "channel": channel, + "id": param_id, + "title": _child_text(param_el, "label"), + "help": _child_text(param_el, "description"), + "section": section, + "required": required, + "imageType": param_el.get("type"), # input type -> accepts + "fileExtensions": param_el.get("fileExtensions"), + "values": values, + "default": _parse_default(widget, _first_child(param_el, "default")), + "constraints": _parse_constraints( + widget, _first_child(param_el, "constraints") + ), + } + + +def _parse_panel(panel_el): + """Group a ```` panel's children by their leading ``