diff --git a/README.md b/README.md index 5bb48b66..03937190 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,22 @@ Computations use CuPy and NVIDIA RAPIDS for performance on large datasets. - **GPU acceleration**: Common single-cell workflows on `AnnData` run on the GPU. - **Ecosystem compatibility**: Works with Scanpy APIs; includes pieces from Squidpy, decoupler, and pertpy. - **Simple installation**: Available via Conda and PyPI. +- **Version-matched agent skill**: Released pip packages include an analysis skill for Codex, Claude, and other agents. + +## Agent skill + +Released packages include the matching model-neutral analysis skill: + +```bash +rapids-singlecell-install-skills --agent codex +rapids-singlecell-install-skills --agent claude +rapids-singlecell-install-skills --agent claude-science # or: --agent agents +rapids-singlecell-check-kernel # before starting Jupyter +``` + +Use `--dest /path/to/skills/rapids-singlecell` for another agent. See the +[installation guide](https://rapids-singlecell.readthedocs.io/en/latest/installation.html#agent-skill) +for installation and preflight details. ## Documentation diff --git a/docs/installation.md b/docs/installation.md index a8959284..dd40fbc6 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -166,6 +166,51 @@ Use multiple architectures when building portable binaries (e.g., for a shared c The `-real` suffix generates device code only (no PTX fallback), which reduces binary size. ``` +## Agent skill + +Released packages contain a matching model-neutral analysis skill. Copy it to +an agent's personal skill directory after installing RSC: + +```bash +rapids-singlecell-install-skills --agent codex +rapids-singlecell-install-skills --agent claude +rapids-singlecell-install-skills --agent claude-science +rapids-singlecell-install-skills --agent agents +rapids-singlecell-install-skills --dest /other/skills/rapids-singlecell +``` + +`--check` compares the installed copy with the bundle in the active package; +`--force` replaces a differing copy. `--print-path` prints the bundled source. +For Claude Science, the installer resolves the active organization from +`~/.claude-science/active-org.json`. + +```bash +rapids-singlecell-install-skills --check --agent codex +``` + +Before starting Jupyter, verify RMM, GPU/CUDA execution, the RSC import, native +extensions, and a representative RSC kernel: + +```bash +rapids-singlecell-check-kernel +rapids-singlecell-check-kernel --mode managed # planned oversubscription +``` + +The preflight is disposable; configure RMM again at the start of the notebook. + +### Bootstrap from a standalone skill + +If RSC is not installed, create a fresh environment from the matching canonical +definition on `main`: + +- [CUDA 12](https://github.com/scverse/rapids_singlecell/blob/main/conda/rsc_rapids_26.04_cuda12.yml) +- [CUDA 13](https://github.com/scverse/rapids_singlecell/blob/main/conda/rsc_rapids_26.04_cuda13.yml) + +```bash +CONDA_CHANNEL_PRIORITY=flexible mamba env create --name rsc \ + --file /path/to/rsc_rapids_26.04_cuda13.yml +``` + ## Docker We also offer Docker containers for `rapids-singlecell`. These containers include all the necessary dependencies, making it even easier to get started with `rapids-singlecell`. diff --git a/pyproject.toml b/pyproject.toml index 4c2c2668..f80e6d75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,8 @@ license = { file = "LICENSE" } authors = [ { name = "Severin Dicks" } ] readme = { file = "README.md", content-type = "text/markdown" } dynamic = [ "version" ] +scripts.rapids-singlecell-install-skills = "rapids_singlecell_skills.install:main" +scripts.rapids-singlecell-check-kernel = "rapids_singlecell_skills.kernel:main" dependencies = [ "anndata>=0.10.0", @@ -152,7 +154,7 @@ local_scheme = "node-and-date" version-file = "src/rapids_singlecell/_version.py" [tool.hatch.build.targets.wheel] -packages = [ "src/rapids_singlecell", "src/testing" ] +packages = [ "src/rapids_singlecell", "src/rapids_singlecell_skills", "src/testing" ] exclude = [ "**/*.cu", "**/*.cuh", "**/*.h" ] [tool.hatch.build.targets.wheel.hooks.scikit-build] diff --git a/src/rapids_singlecell/__init__.py b/src/rapids_singlecell/__init__.py index 9217ac06..095f7e8f 100644 --- a/src/rapids_singlecell/__init__.py +++ b/src/rapids_singlecell/__init__.py @@ -5,6 +5,8 @@ from . import dcg, get, gr, pp, ptg, tl from ._version import __version__ +__all__ = ["dcg", "get", "gr", "pp", "ptg", "tl"] + def _detect_duplicate_installation(): """Warn if multiple rapids_singlecell variants are installed.""" diff --git a/src/rapids_singlecell/dcg.py b/src/rapids_singlecell/dcg.py index dbe01d98..c84cb172 100644 --- a/src/rapids_singlecell/dcg.py +++ b/src/rapids_singlecell/dcg.py @@ -1,3 +1,5 @@ from __future__ import annotations -from .decoupler_gpu import * +from .decoupler_gpu import aucell, mlm, ulm, waggr, zscore + +__all__ = ["aucell", "mlm", "ulm", "waggr", "zscore"] diff --git a/src/rapids_singlecell/get/__init__.py b/src/rapids_singlecell/get/__init__.py index 7f3f8264..64995f67 100644 --- a/src/rapids_singlecell/get/__init__.py +++ b/src/rapids_singlecell/get/__init__.py @@ -8,3 +8,5 @@ # Aggregation imports preprocessing modules that use the names exported above. # isort: split from ._aggregated import aggregate + +__all__ = ["X_to_CPU", "X_to_GPU", "aggregate", "anndata_to_CPU", "anndata_to_GPU"] diff --git a/src/rapids_singlecell/gr.py b/src/rapids_singlecell/gr.py index ce878aaa..31b74470 100644 --- a/src/rapids_singlecell/gr.py +++ b/src/rapids_singlecell/gr.py @@ -1,3 +1,5 @@ from __future__ import annotations -from .squidpy_gpu import * +from .squidpy_gpu import calculate_niche, co_occurrence, ligrec, spatial_autocorr + +__all__ = ["calculate_niche", "co_occurrence", "ligrec", "spatial_autocorr"] diff --git a/src/rapids_singlecell/pp.py b/src/rapids_singlecell/pp.py index 2734023a..dc239cfe 100644 --- a/src/rapids_singlecell/pp.py +++ b/src/rapids_singlecell/pp.py @@ -1,3 +1,50 @@ from __future__ import annotations -from .preprocessing import * +from .preprocessing import ( + bbknn, + calculate_qc_metrics, + filter_cells, + filter_genes, + harmony_integrate, + highly_variable_genes, + log1p, + neighbors, + normalize_pearson_residuals, + normalize_total, + pca, + regress_out, + scale, + scrublet, + scrublet_simulate_doublets, + sqrt, +) +from .preprocessing import ( + filter_highly_variable as filter_highly_variable, +) +from .preprocessing import ( + flag_gene_family as flag_gene_family, +) + +__all__ = [ + "bbknn", + "calculate_qc_metrics", + "filter_cells", + "filter_genes", + "harmony_integrate", + "highly_variable_genes", + "log1p", + "neighbors", + "normalize_pearson_residuals", + "normalize_total", + "pca", + "regress_out", + "scale", + "scrublet", + "scrublet_simulate_doublets", + "sqrt", +] + +__deprecated_exports__ = { + "filter_highly_variable": "Deprecated; do not use in new analyses.", + "flag_gene_family": "Deprecated; do not use in new analyses.", +} diff --git a/src/rapids_singlecell/preprocessing/_neighbors/__init__.py b/src/rapids_singlecell/preprocessing/_neighbors/__init__.py index 00d74bbf..54abee67 100644 --- a/src/rapids_singlecell/preprocessing/_neighbors/__init__.py +++ b/src/rapids_singlecell/preprocessing/_neighbors/__init__.py @@ -279,6 +279,8 @@ def bbknn( of a symmetrical matrix of connectivities. n_pcs Use this many PCs. If `n_pcs==0` and `use_rep is None`, use `.X`. + batch_key + Key in `adata.obs` containing the batch labels to balance. Required. use_rep Use the indicated representation. `'X'` or any key for `.obsm` is valid. If `None`, the representation is chosen automatically: For `.n_vars < 50`, `.X` diff --git a/src/rapids_singlecell/preprocessing/_qc.py b/src/rapids_singlecell/preprocessing/_qc.py index 249ce3a0..ce6e4f43 100644 --- a/src/rapids_singlecell/preprocessing/_qc.py +++ b/src/rapids_singlecell/preprocessing/_qc.py @@ -26,7 +26,7 @@ def calculate_qc_metrics( layer: str = None, ) -> None: """\ - Calculates basic qc Parameters :cite:p:`McCarthy2017`. + Calculate basic quality-control metrics :cite:p:`McCarthy2017`. Calculates number of genes per cell (n_genes) and number of counts per cell (n_counts). Loosely based on calculate_qc_metrics from scanpy :cite:p:`Wolf2018`. Updates :attr:`~anndata.AnnData.obs` and :attr:`~anndata.AnnData.var` with columns with qc data. diff --git a/src/rapids_singlecell/ptg.py b/src/rapids_singlecell/ptg.py index 2358f441..ae098bb1 100644 --- a/src/rapids_singlecell/ptg.py +++ b/src/rapids_singlecell/ptg.py @@ -1,3 +1,37 @@ from __future__ import annotations -from .pertpy_gpu import * +from .pertpy_gpu import ( + Distance, + GuideAssignment, + Mixscale, + Mixscape, +) +from .pertpy_gpu import ( + MeanVar as MeanVar, +) + +__all__ = ["Distance", "GuideAssignment", "Mixscale", "Mixscape"] + +__deprecated_exports__ = { + "MeanVar": "Deprecated; do not use in new analyses.", +} + +# Public class members are part of the installed API contract used by agent tooling. +# Keep these in sync with docs/api/pertpy_gpu.md. +__api_members__ = { + "Distance": [ + "__call__", + "pairwise", + "onesided_distances", + "contrast_distances", + "create_contrasts", + "bootstrap", + ], + "GuideAssignment": [ + "assign_by_threshold", + "assign_to_max_guide", + "assign_mixture_model", + ], + "Mixscale": ["perturbation_signature", "mixscale"], + "Mixscape": ["perturbation_signature", "mixscape", "lda"], +} diff --git a/src/rapids_singlecell/tl.py b/src/rapids_singlecell/tl.py index f14caa4f..4a1bd728 100644 --- a/src/rapids_singlecell/tl.py +++ b/src/rapids_singlecell/tl.py @@ -1,3 +1,42 @@ from __future__ import annotations -from .tools import * +from .tools import ( + diffmap, + draw_graph, + embedding_density, + ingest, + leiden, + louvain, + pca, + rank_genes_groups, + score_genes, + score_genes_cell_cycle, + tsne, + umap, +) +from .tools import ( + kmeans as kmeans, +) +from .tools import ( + rank_genes_groups_logreg as rank_genes_groups_logreg, +) + +__all__ = [ + "diffmap", + "draw_graph", + "embedding_density", + "ingest", + "leiden", + "louvain", + "pca", + "rank_genes_groups", + "score_genes", + "score_genes_cell_cycle", + "tsne", + "umap", +] + +__deprecated_exports__ = { + "kmeans": "Deprecated; do not use in new analyses.", + "rank_genes_groups_logreg": rank_genes_groups_logreg.__deprecated__, +} diff --git a/src/rapids_singlecell_skills/__init__.py b/src/rapids_singlecell_skills/__init__.py new file mode 100644 index 00000000..fa90d305 --- /dev/null +++ b/src/rapids_singlecell_skills/__init__.py @@ -0,0 +1 @@ +"""Agent-skill tools bundled with RAPIDS-singlecell.""" diff --git a/src/rapids_singlecell_skills/api.py b/src/rapids_singlecell_skills/api.py new file mode 100644 index 00000000..45555737 --- /dev/null +++ b/src/rapids_singlecell_skills/api.py @@ -0,0 +1,1313 @@ +"""Query a live, provenance-aware RAPIDS-singlecell API view.""" + +from __future__ import annotations + +import argparse +import importlib +import importlib.metadata +import inspect +import json +import platform +import re +import subprocess +import sys +import textwrap +import tomllib +import typing +from importlib import resources +from pathlib import Path +from types import ModuleType +from typing import Any +from urllib.parse import unquote, urlparse + +_DISTRIBUTION = "rapids-singlecell" +_MAX_RESULTS = 25 +_MIN_SCORE = 8 +_TOKEN_RE = re.compile(r"[a-z0-9]+") +_NAME_RE = re.compile(r"^[A-Za-z_]\w*$") +_SYMBOL_RE = re.compile(r"^[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)+$") +_SPHINX_ROLE_RE = re.compile( + r":(?P[A-Za-z0-9_]+)(?::[A-Za-z0-9_]+)?:`(?P[^`]+)`" +) +_STOPWORDS = { + "a", + "an", + "and", + "can", + "cell", + "cells", + "compute", + "data", + "dataset", + "for", + "find", + "from", + "how", + "in", + "my", + "of", + "on", + "or", + "perform", + "please", + "run", + "should", + "single", + "the", + "to", + "use", + "using", + "with", +} +_ENTRY_KEYS = {"index", "notes"} +_INDEX_KEYS = {"keywords", "related"} +_NOTE_KEYS = {"as_of", "claim", "kind", "source"} + + +class ApiError(RuntimeError): + """Report an unavailable runtime or invalid hand-authored layer.""" + + +class QueryError(ValueError): + """Report an invalid search or description request.""" + + +def _resource_path(name: str) -> Path: + resource = resources.files("rapids_singlecell_skills").joinpath(name) + return Path(str(resource)) + + +def _string_list(value: Any, *, field: str) -> list[str]: + if not isinstance(value, list) or any( + not isinstance(item, str) or not item.strip() for item in value + ): + raise ApiError(f"{field} must be a list of non-empty strings") + if len(set(value)) != len(value): + raise ApiError(f"{field} contains duplicate values") + return value + + +def _load_hand_layer() -> dict[str, dict[str, Any]]: + path = _resource_path("api_notes.toml") + try: + payload = tomllib.loads(path.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError) as error: + raise ApiError(f"cannot read API hand layer at {path}: {error}") from error + + if set(payload) != {"entries", "schema"} or payload.get("schema") != 1: + raise ApiError(f"unsupported API hand-layer schema at {path}") + entries = payload.get("entries") + if not isinstance(entries, dict): + raise ApiError("API hand layer must contain an entries table") + + for symbol, entry in entries.items(): + if not isinstance(symbol, str) or not _SYMBOL_RE.fullmatch(symbol): + raise ApiError(f"invalid hand-layer symbol: {symbol!r}") + if not isinstance(entry, dict) or not entry or not set(entry) <= _ENTRY_KEYS: + raise ApiError(f"invalid hand-layer fields for {symbol}") + + if "index" in entry: + index = entry["index"] + if not isinstance(index, dict) or set(index) != _INDEX_KEYS: + raise ApiError(f"{symbol}.index must contain keywords and related") + _string_list(index["keywords"], field=f"{symbol}.index.keywords") + related = _string_list(index["related"], field=f"{symbol}.index.related") + if any(not _SYMBOL_RE.fullmatch(item) for item in related): + raise ApiError(f"{symbol}.index.related contains an invalid symbol") + + notes = entry.get("notes", []) + if not isinstance(notes, list): + raise ApiError(f"{symbol}.notes must be an array of tables") + for position, note in enumerate(notes): + label = f"{symbol}.notes[{position}]" + if not isinstance(note, dict) or not set(note) <= _NOTE_KEYS: + raise ApiError(f"{label} contains invalid fields") + kind = note.get("kind") + if kind == "runtime": + raise ApiError(f"{label}: kind='runtime' is forbidden; write a probe") + if kind not in {"durable", "snapshot"}: + raise ApiError(f"{label}.kind must be durable or snapshot") + for field in ("claim", "source"): + value = note.get(field) + if not isinstance(value, str) or not value.strip(): + raise ApiError(f"{label}.{field} must be a non-empty string") + source = " ".join(note["source"].casefold().split()) + if source in { + "a model", + "language model", + "language model memory", + "llm", + "llm memory", + "model", + "model memory", + "model's memory", + "the model", + "the model's memory", + }: + raise ApiError(f"{label}.source cannot cite model memory") + as_of = note.get("as_of") + if kind == "snapshot" and not as_of: + raise ApiError(f"{label}: snapshot notes require as_of") + if as_of is not None and ( + not isinstance(as_of, dict) + or not as_of + or any( + not isinstance(key, str) or not isinstance(value, str) or not value + for key, value in as_of.items() + ) + ): + raise ApiError(f"{label}.as_of must map packages to versions") + return entries + + +def _load_rsc() -> Any: + try: + return importlib.import_module("rapids_singlecell") + except Exception as error: + raise ApiError( + "cannot import the active rapids_singlecell package; activate its " + "environment and run rapids-singlecell-check-kernel first: " + f"{type(error).__name__}: {error}" + ) from error + + +def _distribution_version(module_name: str) -> tuple[str, str] | None: + distributions = importlib.metadata.packages_distributions().get(module_name, ()) + found: list[tuple[str, str]] = [] + for distribution in distributions: + try: + found.append((distribution, importlib.metadata.version(distribution))) + except importlib.metadata.PackageNotFoundError: + continue + if not found: + return None + return sorted(found)[0] + + +def _editable_source_root(module: Any) -> Path | None: + source = Path(module.__file__).resolve() + distributions = importlib.metadata.packages_distributions().get( + "rapids_singlecell", () + ) + for name in sorted(distributions): + try: + direct_url = importlib.metadata.distribution(name).read_text( + "direct_url.json" + ) + except importlib.metadata.PackageNotFoundError: + continue + if direct_url is None: + continue + try: + payload = json.loads(direct_url) + parsed = urlparse(payload["url"]) + except (KeyError, TypeError, json.JSONDecodeError): + continue + if payload.get("dir_info", {}).get("editable") is not True: + continue + if parsed.scheme != "file" or parsed.netloc not in {"", "localhost"}: + continue + root = Path(unquote(parsed.path)).resolve() + if source.is_relative_to(root): + return root + + for repository in source.parents: + expected = repository / "src" / "rapids_singlecell" + if (repository / ".git").exists() and expected.resolve() == source.parent: + return repository + return None + + +def _source_state(module: Any) -> tuple[str, bool] | None: + repository = _editable_source_root(module) + if repository is None: + return None + try: + revision = subprocess.run( + ["git", "-C", str(repository), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + timeout=2, + ) + status = subprocess.run( + ["git", "-C", str(repository), "status", "--porcelain"], + check=True, + capture_output=True, + text=True, + timeout=2, + ) + except (OSError, subprocess.SubprocessError): + return None + value = revision.stdout.strip() + return (value, bool(status.stdout.strip())) if value else None + + +def _built_against(rsc: Any) -> dict[str, Any]: + versions: dict[str, Any] = { + "python": platform.python_version(), + "rsc": str(getattr(rsc, "__version__", "unknown")), + } + if distribution := _distribution_version("rapids_singlecell"): + versions["rsc_distribution"] = f"{distribution[0]}=={distribution[1]}" + if source_state := _source_state(rsc): + versions["rsc_source_editable"] = True + versions["rsc_source_revision"] = source_state[0] + versions["rsc_source_dirty"] = source_state[1] + + for label, module_name in ( + ("anndata", "anndata"), + ("cuml", "cuml"), + ("cupy", "cupy"), + ("cuvs", "cuvs"), + ("dask", "dask"), + ("rmm", "rmm"), + ("scanpy", "scanpy"), + ): + if distribution := _distribution_version(module_name): + versions[label] = distribution[1] + continue + loaded = sys.modules.get(module_name) + version = getattr(loaded, "__version__", None) if loaded is not None else None + if version is not None: + versions[label] = str(version) + return versions + + +def _freshness_diagnostics(built_against: dict[str, Any]) -> list[dict[str, str]]: + diagnostics: list[dict[str, str]] = [] + distribution = built_against.get("rsc_distribution") + runtime = built_against.get("rsc") + if isinstance(distribution, str) and "==" in distribution: + distribution_version = distribution.split("==", maxsplit=1)[1] + if runtime != distribution_version: + diagnostics.append( + { + "kind": "generated", + "severity": "warning", + "message": ( + f"imported RSC reports {runtime}, but installed distribution " + f"metadata reports {distribution_version}" + ), + } + ) + if built_against.get("rsc_source_dirty") is True: + diagnostics.append( + { + "kind": "generated", + "severity": "warning", + "message": ( + "the imported RSC source tree is dirty; its revision alone does " + "not identify the observed code" + ), + } + ) + return diagnostics + + +def _exports(module: Any, *, label: str) -> tuple[str, ...]: + exported = getattr(module, "__all__", None) + if not isinstance(exported, (list, tuple)) or not exported: + raise ApiError(f"{label} must publish a non-empty __all__ contract") + if any(not isinstance(name, str) or not name for name in exported): + raise ApiError(f"{label}.__all__ must contain only non-empty strings") + if len(set(exported)) != len(exported): + raise ApiError(f"{label}.__all__ contains duplicate exports") + return tuple(exported) + + +def _facades(rsc: Any) -> list[tuple[str, ModuleType]]: + facades: list[tuple[str, ModuleType]] = [] + for name in _exports(rsc, label="rapids_singlecell"): + value = getattr(rsc, name, None) + if not isinstance(value, ModuleType): + raise ApiError(f"invalid rapids_singlecell.__all__ entry: {name!r}") + _exports(value, label=f"rsc.{name}") + facades.append((name, value)) + return sorted(facades) + + +def _resolve(rsc: Any, symbol: str) -> Any: + value = rsc + try: + for part in symbol.split("."): + if part.startswith("_"): + raise AttributeError(part) + value = getattr(value, part) + except AttributeError as error: + raise QueryError(f"unknown live RSC symbol: {symbol}") from error + if not callable(value): + raise QueryError(f"RSC symbol is not callable: {symbol}") + return value + + +def _kind(value: Any) -> str: + if inspect.isclass(value): + return "class" + if inspect.ismethod(value): + return "method" + if inspect.isfunction(value): + return "function" + return "callable" + + +_ADDRESS_RE = re.compile(r" at 0x[0-9a-fA-F]+>") + + +def _stable_text(value: str) -> str: + value = value.replace("", value) + + +def _annotation(value: Any) -> str: + if value is inspect.Parameter.empty: + return "unspecified" + if isinstance(value, str): + return value + return _stable_text(inspect.formatannotation(value)) + + +def _default(parameter: inspect.Parameter) -> str: + if parameter.kind in { + inspect.Parameter.VAR_KEYWORD, + inspect.Parameter.VAR_POSITIONAL, + }: + return "not applicable" + if parameter.default is inspect.Parameter.empty: + return "required" + return _stable_text(repr(parameter.default)) + + +def _interface( + rsc: Any, symbol: str, value: Any +) -> tuple[str, inspect.Signature, dict[str, Any]]: + try: + signature = inspect.signature(value) + except (TypeError, ValueError) as error: + raise ApiError(f"cannot inspect signature for {symbol}: {error}") from error + parts = symbol.split(".") + if len(parts) != 3: + return _kind(value), signature, {} + + owner_symbol = ".".join(parts[:-1]) + owner = _resolve(rsc, owner_symbol) + descriptor = inspect.getattr_static(owner, parts[-1]) + metadata = {"call": f"rsc.{symbol}", "owner": owner_symbol} + if isinstance(descriptor, staticmethod): + return "staticmethod", signature, metadata + if isinstance(descriptor, classmethod): + return "classmethod", signature, metadata + + parameters = list(signature.parameters.values()) + if not parameters or parameters[0].name not in {"self", "cls"}: + raise ApiError(f"cannot identify the receiver for live method: {symbol}") + signature = signature.replace(parameters=parameters[1:]) + class_name = parts[-2] + instance = re.sub(r"(? str: + doc = inspect.getdoc(value) or "" + lines = doc.splitlines() + stop = len(lines) + for index in range(len(lines) - 1): + underline = lines[index + 1].strip() + if lines[index].strip() and len(underline) >= 3 and set(underline) == {"-"}: + stop = index + break + first = "\n".join(lines[:stop]).split("\n\n", maxsplit=1)[0] + return " ".join(first.split()) + + +def _generated_record( + rsc: Any, + symbol: str, + value: Any, + *, + aliases: list[str] | None = None, +) -> dict[str, Any]: + kind, signature, interface = _interface(rsc, symbol, value) + record: dict[str, Any] = { + "aliases": aliases or [], + "deprecated": getattr(value, "__deprecated__", None), + "kind": kind, + "module": getattr(value, "__module__", None) or type(value).__module__, + "params": [parameter.name for parameter in signature.parameters.values()], + "signature": _stable_text(str(signature)), + "summary": _summary(value), + } + record.update(interface) + return record + + +def _contract(rsc: Any) -> dict[str, Any]: + contract: dict[str, Any] = {} + for namespace, module in _facades(rsc): + exported = _exports(module, label=f"rsc.{namespace}") + public: dict[str, Any] = {} + for name in exported: + value = getattr(module, name, None) + if name.startswith("_") or not callable(value): + raise ApiError(f"invalid rsc.{namespace}.__all__ entry: {name!r}") + public[name] = value + contract[f"{namespace}.{name}"] = value + + class_names = {name for name, value in public.items() if inspect.isclass(value)} + class_members = getattr(module, "__api_members__", {}) + if not isinstance(class_members, dict) or set(class_members) != class_names: + raise ApiError( + f"rsc.{namespace}.__api_members__ must exactly cover exported classes" + ) + for class_name, members in class_members.items(): + _string_list(members, field=f"rsc.{namespace}.{class_name} members") + owner = public[class_name] + for name in members: + try: + value = getattr(owner, name) + except AttributeError as error: + raise ApiError( + "public member contract references absent method: " + f"{namespace}.{class_name}.{name}" + ) from error + if not callable(value): + raise ApiError( + "public member contract is not callable: " + f"{namespace}.{class_name}.{name}" + ) + contract[f"{namespace}.{class_name}.{name}"] = value + return dict(sorted(contract.items())) + + +def _deprecated_contract(rsc: Any) -> dict[str, str]: + deprecated: dict[str, str] = {} + for namespace, module in _facades(rsc): + mapping = getattr(module, "__deprecated_exports__", {}) + if not isinstance(mapping, dict): + raise ApiError(f"rsc.{namespace}.__deprecated_exports__ must be a mapping") + public = set(_exports(module, label=f"rsc.{namespace}")) + overlap = public & set(mapping) + if overlap: + raise ApiError( + f"rsc.{namespace} exports deprecated names as public: " + + ", ".join(sorted(overlap)) + ) + for name, message in mapping.items(): + if ( + not isinstance(name, str) + or not _NAME_RE.fullmatch(name) + or not isinstance(message, str) + or not message.strip() + or not callable(getattr(module, name, None)) + ): + raise ApiError( + f"invalid rsc.{namespace}.__deprecated_exports__ entry: {name!r}" + ) + deprecated[f"{namespace}.{name}"] = message.strip() + return dict(sorted(deprecated.items())) + + +def _surface(rsc: Any, contract: dict[str, Any]) -> list[dict[str, Any]]: + grouped: dict[int, list[tuple[str, Any]]] = {} + methods: list[tuple[str, Any]] = [] + for symbol, value in contract.items(): + if symbol.count(".") == 1: + grouped.setdefault(id(value), []).append((symbol, value)) + else: + methods.append((symbol, value)) + + records: list[dict[str, Any]] = [] + for candidates in grouped.values(): + candidates.sort(key=lambda item: item[0]) + symbol, value = candidates[0] + aliases = [candidate for candidate, _ in candidates[1:]] + records.append( + { + "symbol": symbol, + "value": value, + "generated": _generated_record(rsc, symbol, value, aliases=aliases), + } + ) + + for symbol, value in methods: + records.append( + { + "symbol": symbol, + "value": value, + "generated": _generated_record(rsc, symbol, value), + } + ) + return sorted(records, key=lambda item: item["symbol"]) + + +def _normalize_symbol(requested: str) -> str: + normalized = requested.strip() + for prefix in ("rapids_singlecell.", "rapids-singlecell.", "rsc."): + if normalized.startswith(prefix): + normalized = normalized.removeprefix(prefix) + break + if not (_NAME_RE.fullmatch(normalized) or _SYMBOL_RE.fullmatch(normalized)): + raise QueryError(f"invalid RSC symbol: {requested!r}") + return normalized + + +def _deprecated_match( + deprecated: dict[str, str], requested: str +) -> tuple[str, str] | None: + try: + normalized = _normalize_symbol(requested) + except QueryError: + return None + if normalized in deprecated: + return normalized, deprecated[normalized] + if "." not in normalized: + matches = [ + (symbol, message) + for symbol, message in deprecated.items() + if symbol.rsplit(".", maxsplit=1)[-1] == normalized + ] + if len(matches) == 1: + return matches[0] + return None + + +def _contract_lookup( + rsc: Any, + contract: dict[str, Any], + deprecated: dict[str, str], + requested: str, +) -> dict[str, Any]: + normalized = _normalize_symbol(requested) + if "." not in normalized: + matches = [ + symbol + for symbol in contract + if symbol.rsplit(".", maxsplit=1)[-1] == normalized + ] + if len(matches) == 1: + normalized = matches[0] + elif len(matches) > 1: + choices = ", ".join(matches) + raise QueryError( + f"ambiguous symbol {requested!r}; choose one of: {choices}" + ) + if normalized not in contract: + if match := _deprecated_match(deprecated, normalized): + symbol, message = match + raise QueryError( + f"rsc.{symbol} is deprecated and excluded from the public " + f"contract: {message}" + ) + raise QueryError(f"symbol is not in RSC's live public contract: {requested}") + + value = contract[normalized] + aliases = ( + sorted( + symbol + for symbol, candidate in contract.items() + if symbol != normalized and symbol.count(".") == 1 and candidate is value + ) + if normalized.count(".") == 1 + else [] + ) + return { + "symbol": normalized, + "value": value, + "generated": _generated_record(rsc, normalized, value, aliases=aliases), + } + + +def _validate_hand_symbols( + entries: dict[str, dict[str, Any]], contract: dict[str, Any] +) -> None: + available = set(contract) + for symbol, entry in entries.items(): + if symbol not in available: + raise ApiError(f"hand layer references an absent live symbol: {symbol}") + for related in entry.get("index", {}).get("related", ()): + if related not in available: + raise ApiError( + f"hand layer relation is absent from the live API: {symbol} -> {related}" + ) + + +def _hand_entry( + entries: dict[str, dict[str, Any]], surface_entry: dict[str, Any] +) -> dict[str, Any]: + candidates = [surface_entry["symbol"], *surface_entry["generated"]["aliases"]] + return next((entries[item] for item in candidates if item in entries), {}) + + +def _query_tokens(value: str) -> tuple[str, ...]: + tokens = [ + token + for token in _TOKEN_RE.findall(value.casefold()) + if token not in _STOPWORDS + ] + if not tokens: + raise QueryError("search query must contain an informative word") + return tuple(tokens) + + +def _search_score( + entry: dict[str, Any], hand_entry: dict[str, Any], query: str +) -> int | None: + query_tokens = _query_tokens(query) + generated = entry["generated"] + symbol = entry["symbol"].casefold().replace("_", " ") + basename = symbol.rsplit(".", maxsplit=1)[-1] + aliases = " ".join(generated["aliases"]).casefold().replace("_", " ") + keywords = " ".join(hand_entry.get("index", {}).get("keywords", ())).casefold() + summary = generated["summary"].casefold() + signature = generated["signature"].casefold() + corpus = set( + _TOKEN_RE.findall(" ".join((symbol, aliases, keywords, summary, signature))) + ) + matched_tokens = [token for token in query_tokens if token in corpus] + required_matches = ( + 1 + if len(query_tokens) == 1 or any(token == basename for token in query_tokens) + else 2 + ) + if len(matched_tokens) < required_matches: + return None + + normalized_query = " ".join(query_tokens) + score = 0 + if normalized_query == basename: + score += 100 + if normalized_query in symbol: + score += 40 + if normalized_query in keywords: + score += 60 + if normalized_query in summary: + score += 20 + for token in matched_tokens: + score += 12 if token in basename else 0 + score += 8 if token in keywords else 0 + score += 4 if token in summary else 0 + score += 2 if token in signature else 0 + score -= 2 * (len(query_tokens) - len(matched_tokens)) + return score + + +def search( + surface: list[dict[str, Any]], + entries: dict[str, dict[str, Any]], + query: str, + *, + namespace: str | None = None, + limit: int = 8, +) -> list[dict[str, Any]]: + """Return deterministic matches built from the live surface and sparse index.""" + if not 1 <= limit <= _MAX_RESULTS: + raise QueryError(f"limit must be between 1 and {_MAX_RESULTS}") + namespaces = {entry["symbol"].split(".", maxsplit=1)[0] for entry in surface} + if namespace is not None and namespace not in namespaces: + raise QueryError( + f"unknown live namespace {namespace!r}; choose one of: " + + ", ".join(sorted(namespaces)) + ) + + matches: list[tuple[int, dict[str, Any]]] = [] + prefix = f"{namespace}." if namespace else None + for entry in surface: + selected = entry + if prefix: + paths = [entry["symbol"], *entry["generated"]["aliases"]] + matching = [path for path in paths if path.startswith(prefix)] + if not matching: + continue + symbol = matching[0] + if symbol != entry["symbol"]: + selected = { + **entry, + "generated": dict(entry["generated"]), + "symbol": symbol, + } + selected["generated"]["aliases"] = [ + path for path in paths if path != symbol + ] + if selected["generated"]["deprecated"]: + continue + score = _search_score(selected, _hand_entry(entries, selected), query) + if score is not None and score >= _MIN_SCORE: + matches.append((score, selected)) + matches.sort(key=lambda item: (-item[0], item[1]["symbol"])) + return [entry for _, entry in matches[:limit]] + + +def _split_doc(doc: str) -> dict[str, str]: + lines = doc.splitlines() + headers: list[tuple[int, str]] = [] + for index in range(len(lines) - 1): + underline = lines[index + 1].strip() + if lines[index].strip() and len(underline) >= 3 and set(underline) == {"-"}: + headers.append((index, lines[index].strip())) + sections: dict[str, str] = {} + for position, (index, name) in enumerate(headers): + stop = headers[position + 1][0] if position + 1 < len(headers) else len(lines) + body = lines[index + 2 : stop] + while body and not body[0].strip(): + body.pop(0) + while body and not body[-1].strip(): + body.pop() + sections[name] = "\n".join(body) + return sections + + +def _parameter_docs(section: str) -> dict[str, str]: + lines = section.splitlines() + indents = [len(line) - len(line.lstrip()) for line in lines if line.strip()] + if not indents: + return {} + header_indent = min(indents) + headers = [ + index + for index, line in enumerate(lines) + if line.strip() + and len(line) - len(line.lstrip()) == header_indent + and not line.lstrip().startswith(("-", "*", ":")) + ] + descriptions: dict[str, str] = {} + for position, index in enumerate(headers): + stop = headers[position + 1] if position + 1 < len(headers) else len(lines) + names = lines[index].strip().split(":", maxsplit=1)[0] + description = " ".join(" ".join(lines[index + 1 : stop]).split()) + for name in names.split(","): + descriptions[name.strip().removeprefix("*")] = description + return descriptions + + +def _parameter_records( + signature: inspect.Signature, sections: dict[str, str] +) -> list[dict[str, str]]: + docs = _parameter_docs(sections.get("Parameters", "")) + records = [ + { + "annotation": _annotation(parameter.annotation), + "default": _default(parameter), + "description": docs.get(parameter.name.removeprefix("*"), "undocumented"), + "kind": parameter.kind.name.casefold().replace("_", "-"), + "name": parameter.name, + } + for parameter in signature.parameters.values() + ] + names = {record["name"].removeprefix("*") for record in records} + if any( + parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in signature.parameters.values() + ): + records.extend( + { + "annotation": "unspecified", + "default": "not exposed by signature", + "description": description, + "kind": "forwarded-keyword", + "name": name, + } + for name, description in docs.items() + if name not in names + ) + return records + + +def _merge_notes( + hand_entry: dict[str, Any], built_against: dict[str, Any] +) -> list[dict[str, Any]]: + merged: list[dict[str, Any]] = [] + freshness = _freshness_diagnostics(built_against) + for note in hand_entry.get("notes", ()): + item = dict(note) + if note["kind"] == "snapshot": + mismatches = { + package: {"expected": version, "observed": built_against.get(package)} + for package, version in note["as_of"].items() + if built_against.get(package) != version + } + item["status"] = "suspect" if mismatches or freshness else "current" + if mismatches: + item["mismatches"] = mismatches + if freshness: + item["freshness_diagnostics"] = [ + diagnostic["message"] for diagnostic in freshness + ] + else: + item["status"] = "cited" + merged.append(item) + return merged + + +def _merged_view( + rsc: Any, + surface_entry: dict[str, Any], + hand_entry: dict[str, Any], + built_against: dict[str, Any], + *, + parameter: str | None = None, + section: str | None = None, + full: bool = False, + probed: dict[str, Any] | None = None, +) -> dict[str, Any]: + symbol = surface_entry["symbol"] + value = surface_entry["value"] + kind, signature, _ = _interface(rsc, symbol, value) + doc = inspect.getdoc(value) or "" + sections = _split_doc(doc) + parameters = _parameter_records(signature, sections) + generated = dict(surface_entry["generated"]) + generated["kind"] = kind + generated["params"] = [item["name"] for item in parameters] + generated["choices"] = _parameter_choices(value) + generated["sections"] = sorted(sections) + + if parameter is not None: + requested = parameter.removeprefix("*") + match = next( + ( + item + for item in parameters + if item["name"].removeprefix("*") == requested + ), + None, + ) + if match is None: + available = ", ".join(item["name"] for item in parameters) + raise QueryError( + f"{symbol} has no parameter {parameter!r}; available: {available}" + ) + generated["parameter"] = match + elif section is not None: + match = next( + (name for name in sections if name.casefold() == section.casefold()), None + ) + if match is None: + raise QueryError( + f"{symbol} has no section {section!r}; available: " + + ", ".join(sorted(sections)) + ) + generated["section"] = {"name": match, "body": sections[match]} + elif full: + generated["doc"] = doc + + index = hand_entry.get("index", {"keywords": [], "related": []}) + return { + "built_against": built_against, + "diagnostics": _freshness_diagnostics(built_against), + "generated": generated, + "index": index, + "notes": _merge_notes(hand_entry, built_against), + "probed": probed, + "symbol": symbol, + } + + +def _run_probe(symbol: str) -> dict[str, Any]: + return { + "status": "unavailable", + "reason": ( + f"no safe tiny-input probe is registered for {symbol}; verify the " + "behavior against the active package rather than inferring it" + ), + } + + +def _short_summary(entry: dict[str, Any]) -> str: + value = entry["generated"]["summary"] or "No summary documented." + value = _display_text(value) + return textwrap.shorten(value, width=160, placeholder=" …") + + +def _display_text(value: str) -> str: + def replace(match: re.Match[str]) -> str: + if match.group("role") == "cite": + return "" + target = match.group("value").strip().lstrip("~") + return target.split("<", maxsplit=1)[0].strip() + + rendered = " ".join(_SPHINX_ROLE_RE.sub(replace, value).split()) + return re.sub(r"\s+([,.;:])", r"\1", rendered) + + +def _print_search( + built_against: dict[str, Any], + query: str, + entries: list[dict[str, Any]], + *, + miss_message: str | None = None, +) -> None: + print(f"built against: rsc={built_against['rsc']}") + _print_diagnostics(_freshness_diagnostics(built_against)) + print(f"query: {query}") + if not entries: + print( + miss_message + or "no exact matches; this is a lexical miss, not evidence that RSC " + "lacks the API; retry with fewer or API-specific terms" + ) + return + for entry in entries: + aliases = entry["generated"]["aliases"] + suffix = ( + " (aliases: " + ", ".join(f"rsc.{alias}" for alias in aliases) + ")" + if aliases + else "" + ) + print(f"rsc.{entry['symbol']}{suffix} — {_short_summary(entry)}") + + +def _print_diagnostics(diagnostics: list[dict[str, str]]) -> None: + for diagnostic in diagnostics: + print(f"{diagnostic['severity']}: {diagnostic['message']}") + + +def _print_context(view: dict[str, Any], *, include_index: bool) -> None: + if include_index: + index = view["index"] + if index["keywords"]: + print("keywords: " + ", ".join(index["keywords"])) + if index["related"]: + print("related: " + ", ".join(f"rsc.{item}" for item in index["related"])) + for note in view["notes"]: + print(f"{note['kind']} note [{note['status']}]: {note['claim']}") + print(f" source: {note['source']}") + if as_of := note.get("as_of"): + print(" as_of: " + json.dumps(as_of, sort_keys=True)) + if mismatches := note.get("mismatches"): + print(" mismatches: " + json.dumps(mismatches, sort_keys=True)) + if view["probed"] is not None: + print("probe: " + json.dumps(view["probed"], sort_keys=True)) + + +def _print_view(view: dict[str, Any]) -> None: + generated = view["generated"] + built = view["built_against"] + print(f"built against: rsc={built['rsc']}") + if revision := built.get("rsc_source_revision"): + print(f"source revision: {revision}") + _print_diagnostics(view["diagnostics"]) + print(f"symbol: rsc.{view['symbol']}") + + parameter = generated.get("parameter") + section = generated.get("section") + if parameter is not None: + for key in ("name", "kind", "annotation", "default", "description"): + value = parameter[key] + if key == "description": + value = _display_text(value) + print(f"{key}: {value}") + elif section is not None: + print(f"section: {section['name']}") + print(section["body"]) + else: + if instantiate := generated.get("instantiate"): + if call := instantiate.get("call"): + print(f"instantiate: {call}") + else: + print( + "instantiate: describe rsc." + f"{instantiate['owner']} first; required parameters: " + f"{instantiate['required_parameters']}" + ) + call = generated.get("call", f"rsc.{view['symbol']}") + print(f"signature: {call}{generated['signature']}") + if "doc" in generated: + print(generated["doc"]) + else: + print(f"kind: {generated['kind']}") + summary = generated["summary"] or "Undocumented." + print(f"summary: {_display_text(summary)}") + print("parameters: " + ", ".join(generated["params"])) + for name, members in generated.get("choices", {}).items(): + print(f" {name} choices: {', '.join(members)}") + if deprecated := generated.get("deprecated"): + print(f"deprecated: {deprecated}") + + detail = parameter is not None or section is not None or "doc" in generated + _print_context(view, include_index=not detail) + if not detail: + print("details: use --parameter NAME, --section NAME, --full, or --probe") + + +def _print_json(payload: dict[str, Any]) -> None: + print( + json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + ) + + +_MAP_SUMMARY_CHARS = 68 + + +def _literal_values(resolved: Any) -> list[str]: + """Collect Literal string members, unwrapping PEP 695 aliases and unions.""" + resolved = getattr(resolved, "__value__", resolved) + values = [arg for arg in typing.get_args(resolved) if isinstance(arg, str)] + for arg in typing.get_args(resolved): + arg = getattr(arg, "__value__", arg) + values += [item for item in typing.get_args(arg) if isinstance(item, str)] + return sorted(set(values)) + + +def _parameter_choices(value: Any) -> dict[str, list[str]]: + """Resolve each annotation on its own so one unresolvable name is not fatal. + + ``typing.get_type_hints`` resolves a whole signature at once and raises on the + first ``TYPE_CHECKING``-only name, which hides every other parameter's choices. + Annotations are strings under ``from __future__ import annotations``, so each is + evaluated against its defining module namespace and skipped when that fails. + """ + try: + signature = inspect.signature(value) + except (TypeError, ValueError): + return {} + module = sys.modules.get(getattr(value, "__module__", "") or "") + namespace = {**vars(typing), **(vars(module) if module else {})} + choices: dict[str, list[str]] = {} + for name, parameter in signature.parameters.items(): + annotation = parameter.annotation + if isinstance(annotation, str): + try: + annotation = eval(annotation, namespace) + except Exception: # noqa: BLE001 - an unresolvable name is not an error + continue + if members := _literal_values(annotation): + choices[name] = members + return choices + + +def _map_rows( + contract: dict[str, Any], *, namespace: str | None +) -> list[tuple[str, str]]: + """Return one compact (symbol, summary) row per public symbol.""" + rows = [] + for symbol in sorted(contract): + if namespace is not None and not symbol.startswith(f"{namespace}."): + continue + summary = _display_text(_summary(contract[symbol]) or "") + rows.append( + ( + symbol, + textwrap.shorten( + summary or "No summary documented.", + width=_MAP_SUMMARY_CHARS, + placeholder=" …", + ), + ) + ) + return rows + + +def _print_map( + built_against: dict[str, Any], + rows: list[tuple[str, str]], + *, + contract: dict[str, Any] | None = None, +) -> None: + print(f"built against: rsc={built_against['rsc']}") + print(f"public symbols: {len(rows)}") + width = max((len(symbol) for symbol, _ in rows), default=0) + for symbol, summary in rows: + print(f"rsc.{symbol.ljust(width)} {summary}") + if contract is None: + continue + for name, members in _parameter_choices(contract[symbol]).items(): + print(f"{' ' * (width + 6)}{name}: {', '.join(members)}") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + map_parser = subparsers.add_parser( + "map", help="print the whole public API surface in one pass" + ) + map_parser.add_argument("--namespace") + map_parser.add_argument( + "--options", + action="store_true", + help="also list the accepted values of every enumerated argument", + ) + map_parser.add_argument("--json", action="store_true") + + search_parser = subparsers.add_parser("search", help="search the live RSC API") + search_parser.add_argument("query", nargs="+") + search_parser.add_argument("--namespace") + search_parser.add_argument("--limit", type=int, default=8) + search_parser.add_argument("--json", action="store_true") + + describe_parser = subparsers.add_parser( + "describe", help="build one ephemeral merged API view" + ) + describe_parser.add_argument("symbol") + detail = describe_parser.add_mutually_exclusive_group() + detail.add_argument("--parameter") + detail.add_argument("--section") + detail.add_argument("--full", action="store_true") + describe_parser.add_argument("--probe", action="store_true") + describe_parser.add_argument("--json", action="store_true") + + validate_parser = subparsers.add_parser( + "validate", help="validate the hand layer against the live API" + ) + validate_parser.add_argument("--json", action="store_true") + return parser + + +def main(argv: list[str] | None = None) -> int: + """Run live discovery and build per-symbol ephemeral merged views.""" + parser = _parser() + args = parser.parse_args(argv) + try: + entries = _load_hand_layer() + rsc = _load_rsc() + contract = _contract(rsc) + deprecated = _deprecated_contract(rsc) + _validate_hand_symbols(entries, contract) + built_against = _built_against(rsc) + + if args.command == "validate": + surface = _surface(rsc, contract) + payload = { + "annotated_symbols": len(entries), + "built_against": built_against, + "deprecated_symbols": len(deprecated), + "diagnostics": _freshness_diagnostics(built_against), + "live_symbols": len(surface), + "status": "valid", + } + if args.json: + _print_json(payload) + else: + print( + f"hand layer is valid for {len(surface)} live symbols; " + f"{len(entries)} have sparse annotations" + ) + _print_diagnostics(_freshness_diagnostics(built_against)) + return 0 + + if args.command == "map": + rows = _map_rows(contract, namespace=args.namespace) + if not rows: + raise QueryError(f"no public symbols in namespace {args.namespace!r}") + if args.json: + payload = { + "built_against": built_against["rsc"], + "symbols": {f"rsc.{s}": summary for s, summary in rows}, + } + if args.options: + payload["choices"] = { + f"rsc.{s}": choices + for s, _ in rows + if (choices := _parameter_choices(contract[s])) + } + _print_json(payload) + else: + _print_map( + built_against, rows, contract=contract if args.options else None + ) + return 0 + + if args.command == "search": + surface = _surface(rsc, contract) + query = " ".join(args.query) + matches = search( + surface, + entries, + query, + namespace=args.namespace, + limit=args.limit, + ) + deprecated_match = ( + _deprecated_match(deprecated, query) if not matches else None + ) + miss_message = None + if deprecated_match is not None: + symbol, message = deprecated_match + miss_message = ( + f"rsc.{symbol} is deprecated and excluded from the public " + f"contract: {message}" + ) + if args.json: + payload: dict[str, Any] = { + "built_against": built_against, + "diagnostics": _freshness_diagnostics(built_against), + "query": query, + "results": [ + { + "aliases": [ + f"rsc.{alias}" + for alias in entry["generated"]["aliases"] + ], + "deprecated": entry["generated"]["deprecated"], + "summary": _short_summary(entry), + "symbol": f"rsc.{entry['symbol']}", + } + for entry in matches + ], + } + if not matches: + payload["warning"] = miss_message or ( + "lexical miss; this is not evidence that RSC lacks the API" + ) + _print_json(payload) + else: + _print_search( + built_against, + query, + matches, + miss_message=miss_message, + ) + return 0 + + surface_entry = _contract_lookup(rsc, contract, deprecated, args.symbol) + hand_entry = _hand_entry(entries, surface_entry) + probed = _run_probe(surface_entry["symbol"]) if args.probe else None + view = _merged_view( + rsc, + surface_entry, + hand_entry, + built_against, + parameter=args.parameter, + section=args.section, + full=args.full, + probed=probed, + ) + if args.json: + _print_json(view) + else: + _print_view(view) + return 0 + except QueryError as error: + print(f"error: {error}", file=sys.stderr) + return 2 + except ApiError as error: + print(f"error: {error}", file=sys.stderr) + return 3 + + +__all__ = ["main", "search"] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/rapids_singlecell_skills/api_notes.toml b/src/rapids_singlecell_skills/api_notes.toml new file mode 100644 index 00000000..5dad8623 --- /dev/null +++ b/src/rapids_singlecell_skills/api_notes.toml @@ -0,0 +1,164 @@ +schema = 1 + +[entries."dcg.aucell".index] +keywords = [ + "activity scoring", + "aucell", + "cell-level pathway activity", + "decoupler", + "decoupler gpu", + "gene set enrichment", + "gpu pathway scoring", + "pathway activity", +] +related = [ "dcg.mlm", "dcg.ulm", "dcg.waggr", "dcg.zscore" ] + +[entries."gr.calculate_niche".index] +keywords = [ "niche analysis", "spatial niches", "squidpy", "squidpy gpu" ] +related = [ "gr.co_occurrence", "gr.spatial_autocorr" ] + +[[entries."gr.calculate_niche".notes]] +kind = "snapshot" +claim = "Prefer flavor='cellcharter' as the default: its fixed component count yields an interpretable number of niches, whereas the Leiden-based composition flavors pick their own granularity and can return thousands of micro-niches on a large section. Check niche count and size against data scale whichever flavor you choose." +source = "https://rapids-singlecell.readthedocs.io/en/latest/api/generated/rapids_singlecell.gr.calculate_niche.html" +as_of = { rsc = "0.16.1.dev12+g8810e5dcd.d20260728" } + +[entries."get.aggregate".index] +keywords = [ "aggregate", "aggregation", "grouped summary" ] +related = [ ] + +[entries."get.anndata_to_CPU".index] +keywords = [ "cpu transfer", "host transfer", "move anndata to cpu" ] +related = [ "get.anndata_to_GPU", "get.X_to_CPU" ] + +[entries."get.anndata_to_GPU".index] +keywords = [ "device transfer", "gpu transfer", "move anndata to gpu" ] +related = [ "get.anndata_to_CPU", "get.X_to_GPU" ] + +[entries."pp.bbknn".index] +keywords = [ "batch correction", "batch effect correction", "batch integration", "graph", "knn", "neighbors" ] +related = [ "pp.neighbors", "pp.harmony_integrate" ] + +[[entries."pp.bbknn".notes]] +kind = "durable" +claim = "When a technical covariate is confounded with the biological effect of interest, the two effects are not separately identifiable from those data; adjustment can remove biological signal." +source = "https://www.sc-best-practices.org/cellular_structure/integration" + +[entries."pp.calculate_qc_metrics".index] +keywords = [ "quality control", "quality metrics", "qc", "qc metrics" ] +related = [ "pp.filter_cells", "pp.filter_genes" ] + +[[entries."pp.calculate_qc_metrics".notes]] +kind = "snapshot" +claim = "Inspect multiple QC covariates per sample and prefer permissive, evidence-based outlier thresholds over a universal fixed cutoff." +source = "https://www.sc-best-practices.org/preprocessing_visualization/quality_control.html" +as_of = { rsc = "0.16.1.dev7+g2ec3ef061.d20260722" } + +[entries."pp.harmony_integrate".index] +keywords = [ + "batch correction", + "batch effect correction", + "batch integration", + "corrected pca", + "harmony", + "representation", +] +related = [ "pp.neighbors", "pp.bbknn" ] + +[[entries."pp.harmony_integrate".notes]] +kind = "durable" +claim = "When a technical covariate is confounded with the biological effect of interest, the two effects are not separately identifiable from those data; adjustment can remove biological signal." +source = "https://www.sc-best-practices.org/cellular_structure/integration" + +[entries."pp.highly_variable_genes".index] +keywords = [ + "feature selection", + "highly variable genes", + "hvg", + "m3drop", + "poisson gene selection", + "poisson highly variable genes", +] +related = [ "pp.normalize_pearson_residuals", "pp.pca" ] + +[entries."pp.neighbors".index] +keywords = [ "connectivities", "graph", "knn", "nearest neighbor", "neighbors" ] +related = [ "pp.bbknn", "pp.harmony_integrate", "tl.leiden", "tl.louvain", "tl.umap" ] + +[entries."pp.log1p".index] +keywords = [ "log transform", "log1p", "logarithmize" ] +related = [ "pp.normalize_total" ] + +[entries."pp.normalize_pearson_residuals".index] +keywords = [ "normalize counts", "pearson residuals", "residual normalization" ] +related = [ "pp.normalize_total" ] + +[entries."pp.normalize_total".index] +keywords = [ "library size normalization", "normalize counts", "normalize total", "target sum" ] +related = [ "pp.log1p", "pp.normalize_pearson_residuals" ] + +[entries."pp.pca".index] +keywords = [ "dimensionality reduction", "pca", "principal component analysis", "principal components" ] +related = [ "pp.neighbors" ] + +[entries."pp.scrublet".index] +keywords = [ "detect doublets", "doublet detection", "doublets", "scrublet" ] +related = [ "pp.scrublet_simulate_doublets" ] + +[entries."ptg.Mixscape".index] +keywords = [ "mixscape", "pertpy", "pertpy gpu", "perturbation analysis" ] +related = [ "ptg.Distance", "ptg.GuideAssignment", "ptg.Mixscale" ] + +[entries."tl.leiden".index] +keywords = [ + "cluster", + "clustering", + "community detection", + "float64 clustering", + "leiden", + "neighbor graph", + "reproducible leiden", +] +related = [ "tl.louvain", "pp.neighbors" ] + +[[entries."tl.leiden".notes]] +kind = "snapshot" +claim = "Inspect more than one reasonable resolution and choose granularity using coherent marker evidence rather than treating the default partition as biological truth." +source = "https://www.sc-best-practices.org/cellular_structure/annotation.html" +as_of = { rsc = "0.16.1.dev7+g2ec3ef061.d20260722" } + +[[entries."tl.leiden".notes]] +kind = "snapshot" +claim = "For reproducible Leiden reruns, set dtype='float64' explicitly and hold the input graph, random_state, parameters, and relevant package versions fixed." +source = "https://rapids-singlecell.readthedocs.io/en/latest/api/generated/rapids_singlecell.tl.leiden.html" +as_of = { rsc = "0.16.1.dev8+gbb62f7fcb.d20260723" } + +[entries."tl.louvain".index] +keywords = [ "cluster", "clustering", "community detection", "louvain", "neighbor graph" ] +related = [ "tl.leiden", "pp.neighbors" ] + +[entries."tl.rank_genes_groups".index] +keywords = [ "differential expression", "marker genes", "markers", "rank genes" ] +related = [ ] + +[[entries."tl.rank_genes_groups".notes]] +kind = "durable" +claim = "Cells nested within the same biological sample do not increase the number of independent biological sampling units; cell-level rankings are not replicate-aware differential expression." +source = "https://www.sc-best-practices.org/conditions/differential_gene_expression.html" + +[entries."tl.score_genes_cell_cycle".index] +keywords = [ "cell cycle", "cell cycle scoring", "score cell cycle" ] +related = [ "tl.score_genes" ] + +[entries."tl.tsne".index] +keywords = [ "dimensionality reduction", "embedding", "tsne", "visualization" ] +related = [ ] + +[entries."tl.umap".index] +keywords = [ "dimensionality reduction", "embedding", "umap", "visualization" ] +related = [ "pp.neighbors" ] + +[[entries."tl.umap".notes]] +kind = "durable" +claim = "UMAP axes have no direct meaning; interpret the embedding as a visualization of a selected neighborhood representation, not as a statistical test." +source = "https://www.sc-best-practices.org/preprocessing_visualization/dimensionality_reduction.html" diff --git a/src/rapids_singlecell_skills/data/rapids-singlecell/SKILL.md b/src/rapids_singlecell_skills/data/rapids-singlecell/SKILL.md new file mode 100644 index 00000000..2d1a723c --- /dev/null +++ b/src/rapids_singlecell_skills/data/rapids-singlecell/SKILL.md @@ -0,0 +1,96 @@ +--- +name: rapids-singlecell +description: "GPU single-cell and spatial analysis with RAPIDS-singlecell (rsc) — QC, filtering, normalization, HVG selection, PCA, neighbors, clustering, UMAP, batch integration, differential expression, cell-type annotation, pathway activity scoring, perturbation and CRISPR screens, spatial graphs and niches, and out-of-core Dask runs. Use for any of these, or to set up the runtime: RSC's GPU residency rules, memory routes and API coverage are not guessable from function names." +--- + +# RAPIDS-singlecell + +## Start with the analysis shape + +- For every analysis or notebook request, read [`references/notebooks.md`](references/notebooks.md) before coding. Follow **question → inspect → preserve → compute → evaluate → interpret → export**. + +## Keep GPU data flow explicit + +Importing RSC already installs an RMM-backed CuPy allocator, so explicit configuration is optional. To select a route deliberately, configure RMM before importing CuPy or RSC: + +```python +# Cell 1 — configure RMM exactly once per kernel; never reinitialize again below. +import rmm + +oversubscribe = False +rmm.reinitialize(managed_memory=oversubscribe, pool_allocator=not oversubscribe) +import cupy as cp +from rmm.allocators.cupy import rmm_cupy_allocator + +cp.cuda.set_allocator(rmm_cupy_allocator) +import rapids_singlecell as rsc +import scanpy as sc + +SEED = 0 +``` + +Adapt this shape, one stage per cell, resolving every call against the live package: + +```python +adata = sc.read_h5ad(path) # inspect shape/sparsity/counts before choosing methods +adata.layers["counts"] = adata.X.copy() # preserve immutable source counts +rsc.get.anndata_to_GPU(adata, convert_all=True) # layers too, or pp raises _check_gpu_X +rsc.pp.calculate_qc_metrics(adata, ...) # plot distributions with sc.pl, justify thresholds +rsc.pp.filter_cells(adata, ...); rsc.pp.filter_genes(adata, ...) +rsc.pp.highly_variable_genes(adata, layer="counts", flavor="seurat_v3", n_top_genes=2000) +rsc.pp.normalize_total(adata); rsc.pp.log1p(adata) # normalize the full object +rsc.pp.pca(adata, mask_var="highly_variable") # float32 +rsc.pp.neighbors(adata) +rsc.tl.leiden(adata, dtype="float64", random_state=SEED) # float64 keeps the partition stable +rsc.tl.umap(adata, random_state=SEED) +rsc.tl.rank_genes_groups(adata, groupby="leiden", method="wilcoxon") # annotate from evidence +rsc.get.anndata_to_CPU(adata) # one intentional interop boundary +adata.write(out_path) +``` + +Set `oversubscribe=True` only for managed memory. Before changing allocator mode, read [`references/memory.md`](references/memory.md); for Dask, out-of-core, or multi-GPU work, also read [`references/dask.md`](references/dask.md). + +- Keep GPU preprocessing inputs on-device through the sequence. If a live call raises `_check_gpu_X` or says the input is not a CuPy matrix, inspect `rsc.get.anndata_to_GPU` and move the required data; `convert_all=True` includes layers. Treat this as transitional recovery guidance, not a fixed namespace rule. +- Move data back only at an intentional interop or export boundary with the live `rsc.get.X_to_CPU` or `rsc.get.anndata_to_CPU` signature. Leave already host-backed representations and graphs alone. +- For spatial work, read [`references/spatial.md`](references/spatial.md). Squidpy may supply a physical graph when live RSC cannot; attribute the boundary. + +## Default every computation to RSC + +RSC is the analysis namespace, not an occasional accelerator: `rsc.pp`, `rsc.tl`, `rsc.get`, `rsc.gr`, `rsc.ptg`, `rsc.dcg`. + +- **Print the whole surface before writing code**, so coverage is never a guess: `python -m rapids_singlecell_skills.api map` lists every public symbol with a one-line summary in a single pass; add `--options` for the accepted values of every enumerated argument, or `--namespace get` to narrow. That costs far less than discovering the API a call at a time — do it first, not after a `scanpy` call has already failed. +- To check one specific name, search it: `python -m rapids_singlecell_skills.api search "sc.pp.scale"` returns `rsc.pp.scale`. Prefer the RSC symbol whenever it computes the same result. If the helper is unavailable, use the fallback in [`references/setup.md`](references/setup.md); a miss is not proof of absence. +- Treat a lexical hit as a candidate, not an equivalence; confirm it with `describe`. `sq.gr.spatial_neighbors` returns `rsc.gr.calculate_niche` and `rsc.pp.neighbors`, yet RSC builds no physical spatial graph — never pass an expression kNN graph off as one. +- Only plotting, IO, and named gaps belong outside RSC. When a capability is genuinely absent, state the gap and either stop for a decision or compose the ecosystem method under its own skill at an attributed boundary; never silently substitute CPU computation. +- Before returning the notebook, list every remaining `sc.`/`sq.` computation and justify it. An unjustified one is a defect. +- RSC's public API includes neighboring GPU ports: `rsc.gr` for supported Squidpy-compatible spatial methods, `rsc.ptg` for Pertpy-compatible perturbation methods, and `rsc.dcg` for Decoupler-compatible cell-level scoring. Search both ecosystem and method names, then describe the returned public symbol. + +## Hold these scientific invariants + +These constrain the work rather than being its subject; keep them true without explaining them. + +- Anchor claims to the **biological question and unit of replication**: infer from biological samples, stay descriptive without replication, and keep observation separate from interpretation with contradictions and design limits stated. +- **Inspect the input and uncorrected data first** — count location, labels, batches, design. Keep an uncorrected baseline and batch-correct only an identifiable technical variable crossed with replicated biology. +- Annotate from positive and exclusion programs; use broad, `unknown`, or `mixed` labels with confidence and contradictory evidence when the data do not support a narrower one. +- Respect and accommodate the user's requested analysis choices when compatible with the observed data and design. Push back with evidence when a request is inapplicable or weakens inference, explain why, and propose a valid alternative. +- **Critically evaluate every output before trusting it.** Compare group counts and sizes, figure pixels, render cost, and file size with data scale. Hundreds of megapixels or near-cell-count groups are stop signals: adjust and rerun. Keep the check cheap: read the summary numbers, not every value. + +## Cross package boundaries deliberately + +Every ecosystem boundary is also a device boundary. Cross it explicitly; when unsure where an array lives, inspect `type(adata.X)` rather than defensively re-converting. + +| Crossing | What it needs | +|---|---| +| `sq.gr.spatial_neighbors` → RSC | builds from host coordinates; move the graph on-device | +| `decoupler.op` resources → `rsc.dcg` | host objects; scoring runs on-device | +| RSC → `sc.pl` / `sq.pl` / `pt.pl` | `rsc.get.anndata_to_CPU` first; never hand-roll a converter | + +- For perturbation or CRISPR-screen work, read [`references/perturbation.md`](references/perturbation.md). `rsc.ptg` ports four pertpy APIs; the rest of pertpy stays an attributed boundary. +- For pathway/activity work, obtain resources and pathway-native plots with canonical `decoupler`. Default supported per-cell scoring to a live `rsc.dcg` method and summarize relevant groups descriptively. Aggregate with `rsc.get.aggregate`, then use canonical `decoupler` pseudobulk when requested or for replicated cross-condition inference, preserving biological sample as the replication unit. Attribute each boundary. +- RSC ships no plotting module: plot with the owning package's scverse API — `sc.pl`, `sq.pl`, or `pt.pl` — not hand-rolled `matplotlib`. Use `sc.pl` or `sq.pl` only for embedding, spatial, or other displays that canonical `decoupler` plotting does not express. Make the expression source explicit and disable options that would compute dendrograms, layouts, graphs, smoothing, or transformations. + +## Verify and finish + +- Before Jupyter, run `rapids-singlecell-check-kernel`. For a new or broken runtime, installation/version checks, helper fallback, or preflight failure, read [`references/setup.md`](references/setup.md) and stop until the GPU check passes. +- Preserve immutable input provenance and source counts; execute from a fresh kernel and inspect every rendered output. +- Return the executed `.ipynb`, a continuation-ready AnnData artifact, compact audit tables, and a concise `.md` report of findings, evidence, limitations, and open questions. diff --git a/src/rapids_singlecell_skills/data/rapids-singlecell/agents/openai.yaml b/src/rapids_singlecell_skills/data/rapids-singlecell/agents/openai.yaml new file mode 100644 index 00000000..2b197e94 --- /dev/null +++ b/src/rapids_singlecell_skills/data/rapids-singlecell/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "RAPIDS-singlecell" + short_description: "GPU single-cell and spatial analysis" + default_prompt: "Use $rapids-singlecell for a scientifically justified GPU analysis and return an executed notebook plus a Markdown findings report." diff --git a/src/rapids_singlecell_skills/data/rapids-singlecell/references/dask.md b/src/rapids_singlecell_skills/data/rapids-singlecell/references/dask.md new file mode 100644 index 00000000..056c1c12 --- /dev/null +++ b/src/rapids_singlecell_skills/data/rapids-singlecell/references/dask.md @@ -0,0 +1,26 @@ +# Dask execution + +Use Dask only for capacity, row-chunking, or supported multi-GPU work; it is not the default in-memory route. + +Do not fetch the RSC [out-of-core guide](https://rapids-singlecell.readthedocs.io/en/latest/out_of_core.html) by default: the constraints below plus live API introspection cover routine planning. Open it only when you need worked examples or the current Dask support list. + +This file deliberately duplicates only pre-import orchestration constraints. Treat the worker preflight and the out-of-core guide as authoritative for dependency and transport details. + +## Configure workers + +Multi-GPU is not a flag on a call: it comes from the cluster. Stand up a `LocalCUDACluster` across the devices you intend to use, attach a client, and pass Dask-backed arrays — a method either supports that path or it does not, so confirm support before assuming a call scales out. A few methods shard internally across visible devices without Dask; check the live signature rather than inferring from the name. + +Create and configure GPU workers before the client. Configure RMM and CuPy on each worker; client-process `rmm.reinitialize` does not configure workers. Select transport, allocator, and concurrency only after checking their current compatibility and measuring the workload. + +## Shape the data + +- Follow the out-of-core guide for supported block representations, conversion calls, and chunk layout. Do not infer backend support from an API name or annotation. +- Size chunks from measured worker peak memory and record the chosen layout. Recompute the estimate after filtering when later planning depends on it. +- Treat the co-released out-of-core support list as authoritative; API presence or a docstring alone does not establish Dask support. Confirm the selected signature with `python -m rapids_singlecell_skills.api describe `, and add a scoped probe when behavior matters. +- Stop there for unsupported graphs or embeddings. Materialize only an intentionally reduced object that fits the target device, or stop; never hide a CPU fallback. + +## Compute late + +Keep the graph lazy and call `.compute()` at the last possible step, so no intermediate result is materialized on the way there. Do not call `.compute()` on the full expression matrix merely to build a graph or plot. Estimate the result before `.compute()` or `.persist()`; persist only data that fits across workers. + +On OOM, reduce chunks and task concurrency before adding workers. Re-check the out-of-core guide before changing transport or allocator mode, and return only reduced results. diff --git a/src/rapids_singlecell_skills/data/rapids-singlecell/references/memory.md b/src/rapids_singlecell_skills/data/rapids-singlecell/references/memory.md new file mode 100644 index 00000000..263f6fd5 --- /dev/null +++ b/src/rapids_singlecell_skills/data/rapids-singlecell/references/memory.md @@ -0,0 +1,32 @@ +# Memory management + +Explicit RMM configuration is optional: importing RSC already installs an RMM-backed CuPy allocator. Configure RMM yourself only to select one of the deliberate routes below. + +Do not fetch the RSC [memory guide](https://rapids-singlecell.readthedocs.io/en/latest/memory_management.html) by default: the routes below cover routine configuration. Open it only when a route is ambiguous or when current transport compatibility matters. + +This file deliberately duplicates only decisions that must be made before RSC can be imported and introspected. The disposable preflight is the executable authority; if it disagrees with this checklist, stop and use the memory guide. + +## Choose one route + +| Situation | RMM configuration | +|---|---| +| No deliberate requirement | None; use the allocator RSC installs on import | +| Data and intermediates fit in VRAM | `managed_memory=False`, `pool_allocator=True` | +| Deliberate single-GPU oversubscription | `managed_memory=True`, `pool_allocator=False` | +| Deliberately sized managed pool | Both `True`, with explicit initial and maximum pool sizes | +| Dask or multi-GPU | Configure RMM on each worker; see `dask.md` | + +Managed memory is the oversubscription toggle. It trades speed for capacity. A managed pool oversubscribes only if its maximum can grow beyond VRAM; do not select it without sizing deliberately. Check current transport compatibility in the memory guide. + +## Initialize safely + +- Run `rapids-singlecell-check-kernel` before Jupyter, adding `--mode managed` when that is the intended route. The check is disposable. +- When you do select a route, configure RMM in the notebook before importing CuPy or RSC and before creating GPU arrays. RSC installs an allocator on import, so an explicit choice must come first. +- Point CuPy at `rmm_cupy_allocator`. Never reinitialize while live RMM allocations exist, and never assume configuration crosses process boundaries. +- For Dask, configure `LocalCUDACluster`; do not call client-process `rmm.reinitialize` as a substitute for worker configuration. + +## Plan capacity + +Estimate dense representations, graphs, and temporary workspaces as well as the input matrix. Preserve counts, but plan residency from the next consumer: moving `X` does not imply that layers moved. If GPU preprocessing consumes a layer, move the required data once using the active `anndata_to_GPU` signature; `convert_all` includes supported layers but can enlarge the working set. Inspect the matching `anndata_to_CPU` signature rather than assuming every AnnData slot moves. + +If a pool OOMs, reassess the working set and choose the managed-memory or Dask route deliberately. If managed memory thrashes, reduce the working set or use Dask instead of silently falling back to CPU computation. Verify representation limits against the active stack; do not preserve version-specific limits here. diff --git a/src/rapids_singlecell_skills/data/rapids-singlecell/references/notebooks.md b/src/rapids_singlecell_skills/data/rapids-singlecell/references/notebooks.md new file mode 100644 index 00000000..c9f2972c --- /dev/null +++ b/src/rapids_singlecell_skills/data/rapids-singlecell/references/notebooks.md @@ -0,0 +1,52 @@ +# Analysis notebooks + +Read this file for every analysis or notebook request. + +## Follow this scaffold + +This is a notebook shape, not an API catalog. Resolve calls against the live package and give each numbered item its own Markdown or code cell. + +1. **Markdown — Question and design:** state the biological question, unit of replication, relevant metadata, intended inference, and known limits. +2. **Code — Runtime and provenance:** configure RMM as shown in the core skill, exactly once in this first cell — reinitializing later in the same kernel, while allocations are live, is undefined behavior. Record the seed, input provenance, source revision, and `session_info2`. +3. **Code — Load and inspect:** check shape, sparsity, count location, labels, batches, coordinates, and missing values before choosing methods. +4. **Code — Preserve and place data:** keep immutable input/counts, create the working object, choose the capacity route, and establish required GPU residency. +5. **Code — QC and filter:** plot the relevant distributions with `sc.pl`, justify thresholds, and summarize what was retained; keep the interpretation in the next Markdown cell. +6. **Code — Preprocess:** with `rsc.pp`, select HVGs from counts and normalize/log-transform the full object for a standard workflow. Subset to HVGs only when a later step needs it; otherwise pass `mask_var="highly_variable"` where the call supports it. +7. **Code — Structure:** compute `rsc.pp` PCA and neighbors in float32. Run `rsc.tl` Leiden with `dtype="float64"` by default; for reproducible reruns also hold the graph, observation order, `random_state`, parameters, and package versions fixed. +8. **Code + Markdown — Annotate:** show positive, exclusion, and contradictory marker evidence, tentative labels, confidence, and checkable sources. +9. **Conditional spatial branch:** read [`spatial.md`](spatial.md), build an attributed physical graph, run justified spatial/niche analyses, and inspect granularity. +10. **Code + Markdown — Render and evaluate:** make bounded plots of computed results, sanity-check the consequential outputs against data scale, then interpret them. +11. **Code — Export:** return intended data to host and save AnnData, audit tables, the executed notebook, and the findings report. + +## Make the notebook iterable + +- Organize sections as question → computation → plot or table → interpretation. +- Author the analysis in the notebook itself. If helper scripts exist for batch execution, still write the narrative cells by hand; never assemble a notebook by pasting self-contained scripts into cells, which duplicates each script's imports and runtime setup inside one shared kernel. +- Put one function definition or one analysis task in each code cell, and keep a cell under roughly 25 lines. A stage that needs 90 lines is several tasks, not one. Keep setup, provenance, scientific computation, summarization, plotting, and export separate. +- Put the plot in the next visible cell after preparing the result it renders. Keep its interpretation immediately after it. +- Keep infrastructure compact or hidden, but leave consequential scientific calls visible. Never hide an entire workflow in one omnibus cell. + +## Keep the analysis human + +- Prefer the owning package's native plot for already-computed results: canonical `decoupler` for pathways, `pt.pl` for Pertpy results, and `sc.pl` or `sq.pl` for compatible AnnData or spatial results. Use custom plotting only when no standard ecosystem plot expresses the result. +- Resolve unexpected warnings and suppress only understood routine progress or information logging. Deliver concise outputs without errors or raw log streams. +- Keep preflight, assertion walls, recursive conversion helpers, and package-test scaffolding outside the narrative. Turn useful diagnostics into a compact table or a clearly named validation cell. +- Use `session_info2` for the environment record; it already is the package-version dump, so do not hand-build a second one. Record the seed, input provenance, and source revision alongside it. +- Rebuild tentative labels from current marker evidence after partition changes; never reuse cluster-ID maps. Require complete coverage, assign `unknown` for weak evidence, and show positive, exclusion, contradictory evidence, confidence, and checkable source links. +- Present the work as a scientific analysis, not an RSC-versus-Scanpy benchmark, unless benchmarking was requested. +- Preserve explicit analysis preferences across revisions; verify API-bearing choices in live RSC and keep them visible. Persist one as a standing default only on explicit request, with scope and conditions. +- If a requested choice conflicts with the data or design, show evidence, explain the concern, propose a valid alternative, and never silently ignore or substitute it. + +## Preserve scverse data flow + +- For a standard log-normalized workflow, preserve source counts in a layer, select HVGs from counts, normalize and log-transform the full object, then subset one AnnData copy to HVGs if a later step requires it. +- Before a GPU call consumes a layer, apply the core residency check: moving `X` does not imply that every layer moved. +- Use `rsc.get.X_to_CPU` or `rsc.get.anndata_to_CPU` at an intentional interop boundary; do not invent host-conversion helpers or transfer representations or graphs that are already host-backed. +- Use a non-RSC computation only when requested or essential to the stated question, under its relevant skill and with explicit attribution. State the gap rather than hiding an external computation as an RSC fallback. +- For pathways, obtain resources and pathway-native plots with canonical `decoupler`; default supported per-cell scoring to live `rsc.dcg` and summarize reporting groups. Keep a single sample's group means and cell spread descriptive. Use canonical `decoupler` pseudobulk when requested or for replicated cross-condition inference, aggregating source counts by biological sample and relevant group with `rsc.get.aggregate` before handing off. Load the decoupler skill and attribute each boundary. + +## Finish cleanly + +- Execute every cell from a fresh kernel and inspect the rendered notebook. +- Require every code cell to complete without errors; resolve unexpected warnings and remove only understood routine logs. +- Save the executed `.ipynb`, a continuation-ready AnnData artifact, and compact audit tables. After reviewing them, write a concise `.md` report of findings, evidence, limitations, and unresolved questions. diff --git a/src/rapids_singlecell_skills/data/rapids-singlecell/references/perturbation.md b/src/rapids_singlecell_skills/data/rapids-singlecell/references/perturbation.md new file mode 100644 index 00000000..a25a1cc3 --- /dev/null +++ b/src/rapids_singlecell_skills/data/rapids-singlecell/references/perturbation.md @@ -0,0 +1,35 @@ +# Perturbation analysis + +Read this file for CRISPR screens, perturbation signatures, or distance-based perturbation comparisons. + +## Order the workflow + +- Assign guides with `rsc.ptg.GuideAssignment`, compute the signature with `perturbation_signature`, then run `mixscape` or `mixscale`; `lda` requires `mixscape` first. Each stage raises on a missing predecessor, so confirm `adata.layers["X_pert"]` exists before classifying. +- Run `perturbation_signature` on unscaled log-normalized data. Scaling beforehand corrupts the residual silently — no error is raised — so verify what `X` holds instead of reusing a scaled preprocessing object. +- Set `split_by` to the biological replicate so reference cells come from comparable samples, and keep inference tied to that unit. +- RSC caps `n_neighbors` to the controls available in each split where pertpy raises instead. Note the divergence whenever results are compared against pertpy. + +## Respect the pertpy boundary + +- `rsc.ptg` ports four APIs: `Distance`, `GuideAssignment`, `Mixscape`, and `Mixscale`. `MeanVar` is deprecated; do not use it in new analyses. +- The rest of pertpy has no RSC equivalent — Augur, Milo, Sccoda, Scgen, Cinemaot, Dialogue, the embedding-space classes, the differential-expression wrappers, and the `Distance` significance tests. Compose these under the pertpy skill at an attributed boundary rather than reimplementing them on GPU. +- `Distance` is the only public entry to its metrics: select one with `metric=` and pass metric-specific options through as keyword arguments. Describe the live signature for the supported set rather than assuming a metric name. Every entry point takes `multi_gpu` to fan out across devices. +- Choose the entry point from the question: `pairwise` for all-versus-all, `onesided_distances` for every group against one reference, and `contrast_distances` for an explicit list built with `create_contrasts`, whose `split_by` stratifies each perturbation-versus-control comparison within a cell type, batch, or timepoint. That stratified shape is the usual screen design and the one the other two entry points cannot express. +- Report effect sizes with `bootstrap` uncertainty on `pairwise` or `onesided_distances`. `contrast_distances` takes no `bootstrap` argument, so resample explicitly when a stratified contrast needs an interval. When replicates exist, permute over them rather than over cells; a cell-level permutation treats a single sample's cells as independent and overstates significance. + +## Contrast a screen + +Two steps, and the call sites differ: `create_contrasts` is a staticmethod on the class, while `contrast_distances` needs a configured instance. + +```python +contrasts = rsc.ptg.Distance.create_contrasts( # staticmethod: call on the class + adata, + groupby="target_gene", + selected_group="Non_target", # a sequence compares against several references + split_by="cell_type", # one contrast per perturbation *within* each cell type +) +contrasts = contrasts[contrasts["target_gene"].isin(hits)] # filter before computing +result = rsc.ptg.Distance(metric="edistance").contrast_distances(adata, contrasts) +``` + +`create_contrasts` returns a plain DataFrame — one row per contrast, with the reference in a `reference` column — so inspect and subset it before computing rather than discarding rows afterwards. Combinations whose reference is absent from a split are dropped for you. diff --git a/src/rapids_singlecell_skills/data/rapids-singlecell/references/setup.md b/src/rapids_singlecell_skills/data/rapids-singlecell/references/setup.md new file mode 100644 index 00000000..8d1a221d --- /dev/null +++ b/src/rapids_singlecell_skills/data/rapids-singlecell/references/setup.md @@ -0,0 +1,41 @@ +# Runtime setup and API recovery + +Read this file for a new or broken runtime, installation/version checks, a missing helper, or failed preflight. Keep routine setup outside the notebook. + +## Match the package and skill + +- Treat the installed RSC package and skill as one versioned artifact. For a filesystem copy, run `rapids-singlecell-install-skills --check` with the matching `--agent` (`codex`, `claude`, `claude-science`, or `agents`) or exact `--dest`. Use `python -m rapids_singlecell_skills.install` if the script is unavailable. +- For an uploaded skill, record the active RSC version and source revision; do not claim a package match that cannot be checked. +- Fetch documentation only when actually installing or repairing an environment, not as a routine step. For installation help, use the official [installation guide](https://rapids-singlecell.readthedocs.io/en/latest/installation.html) or the repository's `main` Conda environments for [CUDA 12](https://github.com/scverse/rapids_singlecell/blob/main/conda/rsc_rapids_26.04_cuda12.yml) and [CUDA 13](https://github.com/scverse/rapids_singlecell/blob/main/conda/rsc_rapids_26.04_cuda13.yml). + +## Run disposable preflight + +Run `rapids-singlecell-check-kernel` before Jupyter; if unavailable, run `python -m rapids_singlecell_skills.kernel`. Add `--mode managed` only for intentional oversubscription. Stop on failure. Preflight does not configure the notebook process, so initialize RMM again there before CuPy or RSC imports. + +## Recover from sandbox-blocked kernel transport + +- Use kernel-less execution only after preflight passes and startup logs identify denied Jupyter/ZMQ socket creation. `Kernel died before replying to kernel_info` alone is not diagnostic; investigate import, ABI, GPU, and OOM failures first. +- Use an available, tested in-process notebook executor in one fresh disposable child interpreter—not the agent process. Execute cells in order, stop on first error, and write counts, streams, rich displays, figures, and tracebacks to a notebook copy. Unsupported magics, widgets/comms, or async behavior are blockers. Preserve a scheduler-set `CUDA_VISIBLE_DEVICES`; otherwise set it in the child environment before any CUDA import. If no tested executor is available, report the blocker; claim execution only after persisted outputs are inspected. + +## Discover the live API + +Use a disposable process so failed imports or GPU allocations do not contaminate the notebook: + +```bash +python -m rapids_singlecell_skills.api search "" +python -m rapids_singlecell_skills.api describe --parameter +``` + +Request one parameter or section before `--full`. If the helper CLI and module are unavailable, inspect the installed public callable directly: + +```python +import inspect +import rapids_singlecell as rsc + +call = rsc.pp.highly_variable_genes # replace with the candidate public callable +print(inspect.signature(call)) +print(inspect.getdoc(call)) +help(call) +``` + +Consult the current official documentation next. Inspect active RSC implementation source only when public introspection is insufficient and license compatibility has been verified. A search miss is not proof that a capability is absent. diff --git a/src/rapids_singlecell_skills/data/rapids-singlecell/references/spatial.md b/src/rapids_singlecell_skills/data/rapids-singlecell/references/spatial.md new file mode 100644 index 00000000..0633c6ec --- /dev/null +++ b/src/rapids_singlecell_skills/data/rapids-singlecell/references/spatial.md @@ -0,0 +1,13 @@ +# Spatial analysis + +Read this file for spatial graphs, niches, or large spatial plots. + +## Build the physical graph + +- When live RSC cannot construct the physical spatial graph, use Squidpy at an attributed host-side boundary. Build each sample or library independently, retain coordinate units and parameters, and prevent cross-sample edges. +- Keep physical and expression-derived graphs under distinct keys. Verify observation alignment, degree and component summaries, isolated cells, and edge distances before downstream RSC computation. + +## Bound large spatial plots + +- Prefer `sq.pl` for spatial results. Estimate point and panel counts plus pixels from `figsize × dpi`. Bound figure size and DPI, use `rasterized=True` when supported, and downsample only contextual or grey background points—not analytical data. +- Treat roughly 100–150 DPI as an exploratory heuristic. If render time, pixel count, or file size is disproportionate, stop, simplify, disclose visual sampling, and rerender. diff --git a/src/rapids_singlecell_skills/install.py b/src/rapids_singlecell_skills/install.py new file mode 100644 index 00000000..0dd1fe92 --- /dev/null +++ b/src/rapids_singlecell_skills/install.py @@ -0,0 +1,201 @@ +from __future__ import annotations + +import argparse +import json +import os +import shutil +import sys +from importlib import resources +from pathlib import Path + +SKILL_NAME = "rapids-singlecell" +_DEFAULT_ROOTS = { + "codex": ("CODEX_HOME", "~/.codex"), + "claude": ("CLAUDE_CONFIG_DIR", "~/.claude"), + "agents": (None, "~/.agents"), +} + + +def _claude_science_parent() -> Path: + root = Path("~/.claude-science").expanduser() + active_org_path = root / "active-org.json" + try: + active_org = json.loads(active_org_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ValueError( + "cannot resolve the Claude Science active organization from " + f"{active_org_path}; provide --dest" + ) from error + + org_uuid = active_org.get("org_uuid") if isinstance(active_org, dict) else None + if ( + not isinstance(org_uuid, str) + or not org_uuid + or Path(org_uuid).name != org_uuid + or org_uuid in {".", ".."} + ): + raise ValueError( + f"invalid Claude Science org_uuid in {active_org_path}; provide --dest" + ) + return root / "orgs" / org_uuid / "skills" + + +def skill_source() -> Path: + """Return the package-owned skill directory.""" + source = Path( + str( + resources.files("rapids_singlecell_skills") + .joinpath("data") + .joinpath(SKILL_NAME) + ) + ) + if not (source / "SKILL.md").is_file(): + raise RuntimeError(f"packaged skill is missing: {source}") + return source + + +def _default_parent(agent: str) -> Path: + if agent == "claude-science": + return _claude_science_parent() + try: + variable, fallback = _DEFAULT_ROOTS[agent] + except KeyError as error: + raise ValueError( + f"agent {agent!r} has no default skill directory; provide --dest" + ) from error + root = os.environ.get(variable, fallback) if variable is not None else fallback + return Path(root).expanduser() / "skills" + + +def _target(agent: str, destination: Path | None) -> Path: + if destination is not None: + return destination.expanduser() + return _default_parent(agent) / SKILL_NAME + + +def _snapshot(root: Path) -> dict[Path, bytes | None]: + if root.is_symlink(): + raise RuntimeError(f"refusing symlinked skill directory: {root}") + if not root.is_dir(): + raise RuntimeError(f"skill path is not a directory: {root}") + + snapshot: dict[Path, bytes | None] = {} + for item in root.rglob("*"): + relative = item.relative_to(root) + if item.is_symlink(): + raise RuntimeError(f"refusing symlink in skill directory: {item}") + if item.is_dir(): + snapshot[relative] = None + elif item.is_file(): + snapshot[relative] = item.read_bytes() + else: + raise RuntimeError(f"refusing special file in skill directory: {item}") + return snapshot + + +def _matches(source: Path, target: Path) -> bool: + return _snapshot(source) == _snapshot(target) + + +def _looks_like_existing_skill(target: Path) -> bool: + skill_file = target / "SKILL.md" + try: + text = skill_file.read_text(encoding="utf-8") + except (OSError, UnicodeError): + text = "" + if "name: rapids-singlecell" in text: + return True + anchors = ( + target / "agents" / "openai.yaml", + target / "references" / "dask.md", + target / "references" / "memory.md", + ) + return skill_file.is_file() and all(path.is_file() for path in anchors) + + +def check_skill( + agent: str = "codex", *, destination: Path | None = None +) -> tuple[bool, str]: + """Check whether an agent copy exactly matches the package-owned skill.""" + source = skill_source() + target = _target(agent, destination) + if not target.exists() and not target.is_symlink(): + return False, f"skill is not installed at {target}" + try: + matches = _matches(source, target) + except (OSError, RuntimeError) as error: + return False, str(error) + if not matches: + return False, f"skill differs from the active package: {target}" + return True, f"skill matches the active package: {target}" + + +def install_skill( + agent: str = "codex", + *, + destination: Path | None = None, + force: bool = False, +) -> Path: + """Copy the package-owned skill into an agent's skill directory.""" + source = skill_source() + target = _target(agent, destination) + + if target.is_symlink(): + raise RuntimeError(f"refusing to replace symlinked skill directory: {target}") + if target.exists(): + if target.is_dir() and _matches(source, target): + return target + if not force: + raise RuntimeError(f"destination differs; rerun with --force: {target}") + if target.is_dir(): + if not any(target.iterdir()): + target.rmdir() + elif _looks_like_existing_skill(target): + shutil.rmtree(target) + else: + raise RuntimeError( + "refusing --force for a directory that is not recognizably " + f"the {SKILL_NAME} skill: {target}" + ) + else: + raise RuntimeError(f"refusing --force for non-directory target: {target}") + + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(source, target) + return target + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="rapids-singlecell-install-skills", + description="Install the RAPIDS-singlecell skill bundled with this package.", + ) + parser.add_argument("--agent", default="codex") + parser.add_argument("--dest", type=Path, help="exact destination directory") + parser.add_argument("--force", action="store_true") + action = parser.add_mutually_exclusive_group() + action.add_argument("--check", action="store_true") + action.add_argument("--print-path", action="store_true") + args = parser.parse_args(argv) + + try: + if args.print_path: + print(skill_source()) + return 0 + if args.check: + if args.force: + parser.error("--check cannot be combined with --force") + ok, message = check_skill(args.agent, destination=args.dest) + print(message) + return 0 if ok else 1 + target = install_skill(args.agent, destination=args.dest, force=args.force) + except (OSError, RuntimeError, ValueError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + print(f"Installed RAPIDS-singlecell skill at {target}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/rapids_singlecell_skills/kernel.py b/src/rapids_singlecell_skills/kernel.py new file mode 100644 index 00000000..177302ad --- /dev/null +++ b/src/rapids_singlecell_skills/kernel.py @@ -0,0 +1,248 @@ +"""Check a RAPIDS-singlecell environment before starting a notebook.""" + +from __future__ import annotations + +import argparse +import importlib +import json +import sys +from importlib.machinery import EXTENSION_SUFFIXES +from pathlib import Path +from typing import Any, Literal + +Mode = Literal["pool", "managed", "managed-pool"] +Report = dict[str, Any] + +_MODES = ("pool", "managed", "managed-pool") +_STEPS = ("rmm", "gpu", "cuda", "rsc", "extensions", "api") + + +def _pass(report: Report, step: str, summary: str) -> None: + report["checks"][step] = {"status": "pass", "summary": summary} + + +def _fail(report: Report, step: str, error: BaseException) -> None: + message = " ".join(str(error).splitlines()).strip() or repr(error) + report["checks"][step] = { + "status": "fail", + "summary": message, + "error": {"type": type(error).__name__, "message": message}, + } + + +def _finish(report: Report, *, display: bool) -> Report: + report["ready"] = all( + report["checks"].get(step, {}).get("status") == "pass" for step in _STEPS + ) + if display: + print("RAPIDS-singlecell environment preflight") + for step in _STEPS: + check = report["checks"].get(step, {"status": "skip", "summary": "not run"}) + print(f"{step}: {check['status']} - {check['summary']}") + print("READY" if report["ready"] else "NOT READY") + return report + + +def _rmm_options( + mode: Mode, + device: int, + *, + initial_pool_size: str | None = None, + maximum_pool_size: str | None = None, +) -> dict[str, bool | int | str]: + if mode not in _MODES: + raise ValueError(f"mode must be one of {', '.join(_MODES)}") + if device < 0: + raise ValueError("device must be non-negative") + options: dict[str, bool | int | str] = { + "pool_allocator": mode in {"pool", "managed-pool"}, + "managed_memory": mode in {"managed", "managed-pool"}, + "devices": device, + } + if initial_pool_size is not None: + options["initial_pool_size"] = initial_pool_size + if maximum_pool_size is not None: + options["maximum_pool_size"] = maximum_pool_size + return options + + +def _visible_device_count() -> int: + """Count GPUs without importing CuPy, which must follow RMM configuration.""" + from rmm._cuda.gpu import getDeviceCount + + return int(getDeviceCount()) + + +def _init_rmm( + mode: Mode, + *, + device: int, + initial_pool_size: str | None = None, + maximum_pool_size: str | None = None, +) -> Any: + options = _rmm_options( + mode, + device, + initial_pool_size=initial_pool_size, + maximum_pool_size=maximum_pool_size, + ) + loaded = [name for name in ("cupy", "rapids_singlecell") if name in sys.modules] + if loaded: + raise RuntimeError(f"already imported: {', '.join(loaded)}; start fresh") + + import rmm + + count = _visible_device_count() + if count == 0 or device >= count: + raise RuntimeError(f"device {device} unavailable; {count} GPU(s) visible") + + rmm.reinitialize(**options) + + import cupy as cp + from rmm.allocators.cupy import rmm_cupy_allocator + + cp.cuda.set_allocator(rmm_cupy_allocator) + cp.cuda.Device(device).use() + if cp.cuda.get_allocator() is not rmm_cupy_allocator: + raise RuntimeError("CuPy is not using the RMM allocator") + return cp + + +def _extension_names(directory: Path) -> list[str]: + return sorted( + { + path.name[: -len(suffix)] + for suffix in EXTENSION_SUFFIXES + for path in directory.glob(f"*{suffix}") + if path.name[: -len(suffix)].endswith("_cuda") + } + ) + + +def _preflight( + mode: Mode = "pool", + *, + device: int = 0, + display: bool = True, + initial_pool_size: str | None = None, + maximum_pool_size: str | None = None, +) -> Report: + report: Report = {"checks": {}} + if "ipykernel" in sys.modules: + _fail( + report, + "rmm", + RuntimeError("run before notebook startup; active notebook kernel found"), + ) + return _finish(report, display=display) + + try: + cp = _init_rmm( + mode, + device=device, + initial_pool_size=initial_pool_size, + maximum_pool_size=maximum_pool_size, + ) + _pass(report, "rmm", mode) + except Exception as error: # noqa: BLE001 - this command diagnoses import failures + _fail(report, "rmm", error) + return _finish(report, display=display) + + try: + count = int(cp.cuda.runtime.getDeviceCount()) + if count == 0 or device >= count: + raise RuntimeError(f"device {device} unavailable; {count} GPU(s) visible") + _pass(report, "gpu", f"{count} visible; using device {device}") + + total = cp.arange(16, dtype=cp.int32).sum() + cp.cuda.runtime.deviceSynchronize() + observed = int(total.get()) + if observed != 120: + raise RuntimeError(f"unexpected CuPy result: {observed}") + _pass(report, "cuda", f"CuPy {cp.__version__}; synchronized computation") + except Exception as error: # noqa: BLE001 - this command diagnoses CUDA failures + step = "gpu" if "gpu" not in report["checks"] else "cuda" + _fail(report, step, error) + return _finish(report, display=display) + + try: + rsc = importlib.import_module("rapids_singlecell") + _pass(report, "rsc", f"RAPIDS-singlecell {rsc.__version__}") + except Exception as error: # noqa: BLE001 - this command diagnoses import failures + _fail(report, "rsc", error) + return _finish(report, display=display) + + failures: dict[str, str] = {} + modules: dict[str, Any] = {} + try: + directory = Path(rsc.__file__).resolve().parent / "_cuda" + names = _extension_names(directory) + if not names: + raise RuntimeError("no installed native _cuda extensions found") + for name in names: + try: + modules[name] = importlib.import_module( + f"rapids_singlecell._cuda.{name}" + ) + except Exception as error: # noqa: BLE001 - report every loader failure + failures[name] = " ".join(str(error).splitlines()) + if failures: + raise RuntimeError(f"failed to load: {', '.join(failures)}") + norm = modules.get("_norm_cuda") + if norm is None: + raise RuntimeError("_norm_cuda is not installed") + values = cp.asarray([[1.0, 3.0]], dtype=cp.float32) + norm.mul_dense( + values, + nrows=1, + ncols=2, + target_sum=8.0, + stream=cp.cuda.get_current_stream().ptr, + ) + cp.cuda.runtime.deviceSynchronize() + observed = cp.asnumpy(values).ravel().tolist() + if observed != [2.0, 6.0]: + raise RuntimeError(f"unexpected _norm_cuda result: {observed}") + _pass(report, "extensions", f"{len(names)} loaded; _norm_cuda succeeded") + except Exception as error: # noqa: BLE001 - this command diagnoses kernel failures + _fail(report, "extensions", error) + + try: + from rapids_singlecell_skills import api + + contract = api._contract(rsc) + api._validate_hand_symbols(api._load_hand_layer(), contract) + _pass(report, "api", f"discovery ready; {len(contract)} public symbols") + except Exception as error: # noqa: BLE001 - this command diagnoses helper failures + _fail(report, "api", error) + + return _finish(report, display=display) + + +def main(argv: list[str] | None = None) -> int: + """Run the disposable preflight.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=_MODES, default="pool") + parser.add_argument("--device", type=int, default=0) + parser.add_argument("--initial-pool-size", help="e.g. 1GiB; managed-pool or pool") + parser.add_argument("--maximum-pool-size", help="e.g. 32GiB; managed-pool or pool") + parser.add_argument("--json", action="store_true") + args = parser.parse_args(argv) + + report = _preflight( + args.mode, + device=args.device, + display=not args.json, + initial_pool_size=args.initial_pool_size, + maximum_pool_size=args.maximum_pool_size, + ) + if args.json: + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 if report["ready"] else 1 + + +__all__ = ["main"] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_install_skills.py b/tests/test_install_skills.py new file mode 100644 index 00000000..980a3a24 --- /dev/null +++ b/tests/test_install_skills.py @@ -0,0 +1,760 @@ +from __future__ import annotations + +import json +import re +import sys +import tomllib +from importlib.machinery import EXTENSION_SUFFIXES +from pathlib import Path +from types import ModuleType + +import pytest + +from rapids_singlecell_skills import install, kernel + +ROOT = Path(__file__).parents[1] + + +def test_skill_bundle_is_minimal() -> None: + source = install.skill_source() + assert source == ( + Path(install.__file__).resolve().parent / "data" / "rapids-singlecell" + ) + files = { + path.relative_to(source).as_posix() + for path in source.rglob("*") + if path.is_file() + } + assert files == { + "SKILL.md", + "agents/openai.yaml", + "references/dask.md", + "references/memory.md", + "references/notebooks.md", + "references/perturbation.md", + "references/setup.md", + "references/spatial.md", + } + + text = (source / "SKILL.md").read_text(encoding="utf-8") + assert len(text.splitlines()) <= 100 + assert len(text.split()) <= 1150 + description = re.search(r"(?m)^description:\s*(.+)$", text) + assert description is not None + assert len(description.group(1).strip("\"'")) <= 500 + normalized_text = " ".join(text.split()) + for phrase in ( + "rapids-singlecell-check-kernel", + "managed_memory=oversubscribe", + "references/memory.md", + "references/dask.md", + "references/notebooks.md", + "references/perturbation.md", + "references/setup.md", + "references/spatial.md", + "executed `.ipynb`", + ): + assert phrase in normalized_text + + for name in ( + "memory.md", + "dask.md", + "notebooks.md", + "perturbation.md", + "setup.md", + "spatial.md", + ): + reference = (source / "references" / name).read_text(encoding="utf-8") + assert len(reference.splitlines()) <= 100 + assert len(reference.split()) <= 1000 + + +def _markdown_bullets(path: Path) -> list[str]: + text = path.read_text(encoding="utf-8").casefold() + return [ + " ".join(match.split()) + for match in re.findall( + r"(?ms)^- (.*?)(?=^- |^## |\Z)", + text, + ) + ] + + +def _notebook_contract_bullets() -> list[str]: + return _markdown_bullets(install.skill_source() / "references" / "notebooks.md") + + +def _spatial_contract_bullets() -> list[str]: + return _markdown_bullets(install.skill_source() / "references" / "spatial.md") + + +def _perturbation_contract_bullets() -> list[str]: + return _markdown_bullets(install.skill_source() / "references" / "perturbation.md") + + +def _covers(bullets: list[str], *terms: str) -> bool: + return any(all(term.casefold() in bullet for term in terms) for bullet in bullets) + + +def test_core_preserves_cold_run_recovery_contract() -> None: + bullets = _markdown_bullets(install.skill_source() / "SKILL.md") + + assert _covers( + bullets, + "_check_gpu_x", + "cupy", + "layer", + "rsc.get.anndata_to_gpu", + "convert_all=true", + "transitional", + ) + assert _covers( + bullets, + "squidpy", + "physical graph", + "live rsc cannot", + "attribute", + "references/spatial.md", + ) + assert _covers( + bullets, + "critically evaluate every output", + "group counts", + "figure pixels", + "megapixels", + "stop signals", + "rerun", + ) + assert _covers( + bullets, + "requested analysis choices", + "push back with evidence", + "valid alternative", + ) + assert _covers( + bullets, + "sc.pl", + "sq.pl", + "only", + "canonical `decoupler` plotting does not express", + ) + + +def test_core_dispatches_pathway_work_by_analysis_unit() -> None: + bullets = _markdown_bullets(install.skill_source() / "SKILL.md") + + assert _covers( + bullets, + "canonical", + "decoupler", + "resources", + "pathway-native plots", + "`rsc.dcg`", + "per-cell scoring", + "descriptively", + "pseudobulk", + "replicated cross-condition inference", + "biological sample", + "replication unit", + "attribute each boundary", + ) + assert _covers( + bullets, + "sc.pl", + "sq.pl", + "only", + "canonical `decoupler` plotting does not express", + ) + + +def test_core_surfaces_neighboring_gpu_ports() -> None: + bullets = _markdown_bullets(install.skill_source() / "SKILL.md") + + assert _covers( + bullets, + "`rsc.gr`", + "squidpy-compatible", + "`rsc.ptg`", + "pertpy-compatible", + "`rsc.dcg`", + "decoupler-compatible", + "ecosystem", + "method names", + "describe", + "public symbol", + ) + + +def test_setup_api_discovery_degrades_without_helper() -> None: + setup = ( + (install.skill_source() / "references" / "setup.md") + .read_text(encoding="utf-8") + .casefold() + ) + discovery = " ".join(setup.split("## discover the live api", maxsplit=1)[1].split()) + + for phrase in ( + "rapids_singlecell_skills.api search", + "rapids_singlecell_skills.api describe", + "unavailable", + "inspect.signature", + "inspect.getdoc", + "help(call)", + "official documentation", + "implementation source", + "license compatibility", + ): + assert phrase in discovery + + +def test_setup_bounds_kernel_less_execution_fallback() -> None: + bullets = _markdown_bullets(install.skill_source() / "references" / "setup.md") + + assert _covers( + bullets, + "kernel-less", + "preflight passes", + "startup logs", + "denied jupyter/zmq socket", + "not diagnostic", + "import", + "abi", + "gpu", + "oom", + ) + assert _covers( + bullets, + "tested in-process notebook executor", + "fresh disposable child", + "not the agent process", + "in order", + "stop on first error", + "counts", + "streams", + "rich displays", + "figures", + "tracebacks", + "unsupported magics", + "blockers", + "`cuda_visible_devices`", + "before any cuda import", + "persisted outputs", + ) + + +def test_notebook_reference_preserves_notebook_contract() -> None: + bullets = _notebook_contract_bullets() + + assert _covers(bullets, "one", "analysis task", "code cell") + assert _covers(bullets, "plot", "next visible cell", "interpretation") + assert _covers( + bullets, + "owning package", + "canonical", + "decoupler", + "pathways", + "sc.pl", + "sq.pl", + "custom plotting", + ) + assert _covers( + bullets, + "resolve unexpected warnings", + "understood routine", + "without errors", + "raw log streams", + ) + assert _covers( + bullets, + "scaffolding", + "outside the narrative", + "compact table", + "validation cell", + ) + assert _covers(bullets, "execute every cell", "fresh kernel") + assert _covers( + bullets, + "tentative labels", + "partition changes", + "never reuse cluster-id maps", + "complete coverage", + "`unknown`", + "exclusion", + "contradictory", + "confidence", + "source links", + ) + assert _covers( + bullets, + "analysis preferences", + "live rsc", + "visible", + "standing default", + "explicit request", + "scope", + "conditions", + ) + assert _covers( + bullets, + "executed", + ".ipynb", + ".md", + "findings", + "evidence", + "limitations", + ) + assert _covers( + bullets, + "requested choice", + "data or design", + "evidence", + "concern", + "valid alternative", + "never silently", + ) + + +def test_notebook_reference_guards_kernel_level_reproducibility() -> None: + """Regression guards for the run3 notebook defects (6x RMM re-init, 95-line cells).""" + bullets = _notebook_contract_bullets() + outline = ( + (install.skill_source() / "references" / "notebooks.md") + .read_text(encoding="utf-8") + .casefold() + ) + + assert "exactly once in this first cell" in " ".join(outline.split()) + assert "undefined behavior" in outline + assert _covers( + bullets, + "author the analysis in the notebook", + "never assemble a notebook by pasting", + "shared kernel", + ) + assert _covers(bullets, "under roughly 25 lines", "several tasks, not one") + + +def test_core_shows_a_worked_skeleton() -> None: + """The skeleton is the shape agents copy; prose alone let run3 drift into scripts.""" + text = (install.skill_source() / "SKILL.md").read_text(encoding="utf-8") + + for phrase in ( + "configure RMM exactly once per kernel", + "one stage per cell", + "rsc.get.anndata_to_GPU(adata, convert_all=True)", + 'rsc.tl.leiden(adata, dtype="float64", random_state=SEED)', + "rsc.get.anndata_to_CPU(adata)", + ): + assert phrase in text + + +def test_notebook_reference_preserves_scverse_data_flow() -> None: + bullets = _notebook_contract_bullets() + + assert _covers( + bullets, + "session_info2", + "seed", + "input provenance", + "source revision", + ) + assert _covers( + bullets, + "standard log-normalized workflow", + "counts", + "hvgs", + "full object", + "subset", + ) + assert _covers( + bullets, + "gpu call", + "layer", + "moving `x`", + "every layer", + ) + assert _covers( + bullets, + "rsc.get.x_to_cpu", + "rsc.get.anndata_to_cpu", + "interop", + "host-backed", + ) + assert _covers( + bullets, + "non-rsc", + "essential", + "relevant skill", + "attribution", + "fallback", + ) + + +def test_notebook_reference_routes_decoupler_work() -> None: + bullets = _notebook_contract_bullets() + + assert _covers( + bullets, + "canonical `decoupler`", + "resources", + "pathway-native plots", + "per-cell scoring", + "live `rsc.dcg`", + "single sample", + "descriptive", + "pseudobulk", + "replicated cross-condition inference", + "source counts", + "biological sample", + "decoupler skill", + "attribute each boundary", + ) + + +def test_spatial_reference_bounds_large_spatial_plots() -> None: + bullets = _spatial_contract_bullets() + + assert _covers( + bullets, + "figsize", + "dpi", + "rasterized=true", + "background", + "analytical data", + ) + assert _covers( + bullets, + "render time", + "pixel count", + "file size", + "stop", + "rerender", + ) + + +def test_perturbation_reference_orders_workflow_and_bounds_pertpy() -> None: + bullets = _perturbation_contract_bullets() + + assert _covers( + bullets, + "rsc.ptg.guideassignment", + "perturbation_signature", + "mixscape", + "mixscale", + "`lda` requires", + "x_pert", + ) + assert _covers( + bullets, + "unscaled log-normalized", + "scaling beforehand", + "no error is raised", + ) + assert _covers(bullets, "split_by", "biological replicate") + assert _covers( + bullets, + "`distance`", + "`guideassignment`", + "`mixscape`", + "`mixscale`", + "meanvar", + "deprecated", + ) + assert _covers( + bullets, + "no rsc equivalent", + "pertpy skill", + "attributed boundary", + "reimplementing", + ) + assert _covers(bullets, "permute over", "rather than over cells", "significance") + assert _covers( + bullets, + "`pairwise`", + "`onesided_distances`", + "`contrast_distances`", + "create_contrasts", + "split_by", + "stratifies", + ) + # contrast_distances has no bootstrap parameter; the skill must not imply it does + assert _covers( + bullets, "`contrast_distances` takes no `bootstrap` argument", "resample" + ) + + # the two-step call shape: staticmethod on the class, then a configured instance + text = ( + (install.skill_source() / "references" / "perturbation.md") + .read_text(encoding="utf-8") + .casefold() + ) + for phrase in ( + "rsc.ptg.distance.create_contrasts(", + "staticmethod: call on the class", + 'split_by="cell_type"', + 'rsc.ptg.distance(metric="edistance").contrast_distances(adata, contrasts)', + ): + assert phrase in text + + +def test_notebook_reference_has_ordered_workflow_outline() -> None: + text = ( + (install.skill_source() / "references" / "notebooks.md") + .read_text(encoding="utf-8") + .casefold() + ) + outline = text.split("## follow this scaffold", maxsplit=1)[1].split( + "\n## ", maxsplit=1 + )[0] + items = [ + " ".join(match.split()) + for match in re.findall( + r"(?ms)^\d+\. (.*?)(?=^\d+\. |\Z)", + outline, + ) + ] + + assert len(items) >= 10 + assert any("markdown" in item for item in items) + assert any("code" in item for item in items) + + stages = ( + ("question", "unit of replication", "design"), + ("runtime", "provenance", "session_info2"), + ("load", "inspect", "count location"), + ("preserve", "gpu residency"), + ("qc", "filter"), + ("preprocess", "hvgs", "normalize"), + ("structure", "graph", "leiden"), + ("annotate", "marker evidence"), + ("spatial", "niche"), + ("render", "sanity-check"), + ("export", "anndata", "findings report"), + ) + indices = [ + next( + index + for index, item in enumerate(items) + if all(term in item for term in terms) + ) + for terms in stages + ] + assert indices == sorted(indices) + structure = next( + item + for item in items + if all(term in item for term in ("structure", "graph", "leiden")) + ) + for term in ( + "pca and neighbors in float32", + "`random_state`", + '`dtype="float64"`', + "observation order", + "parameters", + "package versions", + ): + assert term in structure + + +def test_api_index_finds_explicit_method_preferences() -> None: + notes_path = ROOT / "src" / "rapids_singlecell_skills" / "api_notes.toml" + with notes_path.open("rb") as handle: + entries = tomllib.load(handle)["entries"] + + hvg_index = entries["pp.highly_variable_genes"]["index"] + assert "poisson gene selection" in hvg_index["keywords"] + + dcg_index = entries["dcg.aucell"]["index"] + assert "cell-level pathway activity" in dcg_index["keywords"] + assert "decoupler" in dcg_index["keywords"] + + assert "squidpy" in entries["gr.calculate_niche"]["index"]["keywords"] + assert "pertpy" in entries["ptg.Mixscape"]["index"]["keywords"] + + leiden = entries["tl.leiden"] + assert "reproducible leiden" in leiden["index"]["keywords"] + assert any( + note["kind"] == "snapshot" + and "dtype='float64'" in note["claim"] + and "random_state" in note["claim"] + and "input graph" in note["claim"] + for note in leiden["notes"] + ) + + +def test_parameter_choices_resolve_aliased_literals() -> None: + """Choices must survive private aliases and PEP 695 `type` statements. + + `typing.get_type_hints` resolves a whole signature at once and dies on the first + TYPE_CHECKING-only name, which would hide every parameter on these functions. + """ + rsc = pytest.importorskip("rapids_singlecell") + api = pytest.importorskip("rapids_singlecell_skills.api") + + # `flavor: flavors` -- a module-level alias, not an inline Literal + hvg = api._parameter_choices(rsc.pp.highly_variable_genes) + assert "seurat_v3" in hvg["flavor"] + assert "poisson_gene_selection" in hvg["flavor"] + + # `method: _Method | None` -- a PEP 695 `type` alias inside a union + ranked = api._parameter_choices(rsc.tl.rank_genes_groups) + assert {"wilcoxon", "logreg", "t-test"} <= set(ranked["method"]) + + # an inline Literal still works, and unresolvable annotations are skipped + assert "cellcharter" in api._parameter_choices(rsc.gr.calculate_niche)["flavor"] + + +def test_install_check_and_force(tmp_path: Path) -> None: + destination = tmp_path / "rapids-singlecell" + assert install.install_skill(destination=destination) == destination + assert install.check_skill(destination=destination)[0] + + skill_file = destination / "SKILL.md" + skill_file.write_text("modified\n", encoding="utf-8") + assert not install.check_skill(destination=destination)[0] + with pytest.raises(RuntimeError, match="destination differs"): + install.install_skill(destination=destination) + + install.install_skill(destination=destination, force=True) + assert install.check_skill(destination=destination)[0] + + +def test_install_refuses_symlink(tmp_path: Path) -> None: + real = tmp_path / "real" + real.mkdir() + destination = tmp_path / "rapids-singlecell" + destination.symlink_to(real, target_is_directory=True) + with pytest.raises(RuntimeError, match="symlink"): + install.install_skill(destination=destination, force=True) + + +@pytest.mark.parametrize( + ("agent", "directory"), + [("codex", ".codex"), ("claude", ".claude"), ("agents", ".agents")], +) +def test_default_agent_destinations( + agent: str, directory: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.delenv("CODEX_HOME", raising=False) + monkeypatch.delenv("CLAUDE_CONFIG_DIR", raising=False) + assert install._target(agent, None) == ( + tmp_path / directory / "skills" / "rapids-singlecell" + ) + + +def test_default_claude_science_destination( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + root = tmp_path / ".claude-science" + root.mkdir() + (root / "active-org.json").write_text( + json.dumps({"org_uuid": "test-org"}), + encoding="utf-8", + ) + + assert install._target("claude-science", None) == ( + root / "orgs" / "test-org" / "skills" / "rapids-singlecell" + ) + + +def test_claude_science_requires_safe_active_org( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + root = tmp_path / ".claude-science" + root.mkdir() + + with pytest.raises(ValueError, match="active organization"): + install._target("claude-science", None) + + (root / "active-org.json").write_text( + json.dumps({"org_uuid": "../escape"}), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="invalid Claude Science org_uuid"): + install._target("claude-science", None) + + +def test_custom_agent_requires_destination(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="provide --dest"): + install.install_skill("other") + destination = tmp_path / "custom" + assert install.install_skill("other", destination=destination) == destination + + +def test_installer_cli(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + destination = tmp_path / "rapids-singlecell" + assert install.main(["--dest", str(destination)]) == 0 + assert install.main(["--check", "--dest", str(destination)]) == 0 + assert "matches the active package" in capsys.readouterr().out + + +def test_managed_memory_toggle() -> None: + assert kernel._rmm_options("managed", 2) == { + "pool_allocator": False, + "managed_memory": True, + "devices": 2, + } + + +def test_managed_pool_route_is_checkable() -> None: + """memory.md documents a sized managed pool, so preflight must be able to test it.""" + assert kernel._rmm_options( + "managed-pool", 0, initial_pool_size="1GiB", maximum_pool_size="8GiB" + ) == { + "pool_allocator": True, + "managed_memory": True, + "devices": 0, + "initial_pool_size": "1GiB", + "maximum_pool_size": "8GiB", + } + assert "api" in kernel._STEPS + + +def test_preflight_reports_setup_failure(monkeypatch: pytest.MonkeyPatch) -> None: + def fail(*args: object, **kwargs: object) -> None: + raise RuntimeError("broken allocator") + + monkeypatch.delitem(sys.modules, "ipykernel", raising=False) + monkeypatch.setattr(kernel, "_init_rmm", fail) + report = kernel._preflight(display=False) + assert not report["ready"] + assert report["checks"]["rmm"]["status"] == "fail" + + +def test_preflight_refuses_notebook(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "ipykernel", ModuleType("ipykernel")) + report = kernel._preflight(display=False) + assert not report["ready"] + assert "before notebook startup" in report["checks"]["rmm"]["summary"] + + +def test_extension_discovery(tmp_path: Path) -> None: + suffix = EXTENSION_SUFFIXES[0] + (tmp_path / f"_norm_cuda{suffix}").touch() + (tmp_path / f"helper{suffix}").touch() + assert kernel._extension_names(tmp_path) == ["_norm_cuda"] + + +def test_kernel_json_cli( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + report = {"ready": True, "checks": {}} + monkeypatch.setattr(kernel, "_preflight", lambda *args, **kwargs: report) + assert kernel.main(["--json"]) == 0 + assert json.loads(capsys.readouterr().out) == report + + +def test_console_scripts_are_packaged() -> None: + with (ROOT / "pyproject.toml").open("rb") as handle: + project = tomllib.load(handle) + assert project["project"]["scripts"] == { + "rapids-singlecell-install-skills": "rapids_singlecell_skills.install:main", + "rapids-singlecell-check-kernel": "rapids_singlecell_skills.kernel:main", + } + assert ( + "src/rapids_singlecell_skills" + in project["tool"]["hatch"]["build"]["targets"]["wheel"]["packages"] + )