From 5c313087e24f49bf8f2eb86e29ec844172b67053 Mon Sep 17 00:00:00 2001 From: seanmcculloch Date: Tue, 15 Sep 2026 15:36:39 -0700 Subject: [PATCH 1/3] feat(mesoscope): add native schema v2 mapping --- README.md | 1 + .../source/aind_metadata_mapper.mesoscope.rst | 12 +- src/aind_metadata_mapper/mapper_registry.py | 2 + src/aind_metadata_mapper/mesoscope/README.md | 62 ++ .../mesoscope/__init__.py | 5 + src/aind_metadata_mapper/mesoscope/mapper.py | 600 ++++++++++++++++++ tests/fixtures/mesoscope_intermediate.json | 118 ++++ tests/mesoscope/test_mapper.py | 541 ++++++++++++++++ 8 files changed, 1331 insertions(+), 10 deletions(-) create mode 100644 src/aind_metadata_mapper/mesoscope/README.md create mode 100644 src/aind_metadata_mapper/mesoscope/__init__.py create mode 100644 src/aind_metadata_mapper/mesoscope/mapper.py create mode 100644 tests/fixtures/mesoscope_intermediate.json create mode 100644 tests/mesoscope/test_mapper.py diff --git a/README.md b/README.md index 6d5a0be5..91fbb08d 100644 --- a/README.md +++ b/README.md @@ -205,3 +205,4 @@ acquisition.write_standard_file(output_directory=job_settings.output_directory, #### Individual mappers [FIP (Fiber photometry)](src/aind_metadata_mapper/fip/README.md) +[Mesoscope](src/aind_metadata_mapper/mesoscope/README.md) diff --git a/docs/source/aind_metadata_mapper.mesoscope.rst b/docs/source/aind_metadata_mapper.mesoscope.rst index 62dde656..bd94c11f 100644 --- a/docs/source/aind_metadata_mapper.mesoscope.rst +++ b/docs/source/aind_metadata_mapper.mesoscope.rst @@ -4,18 +4,10 @@ aind\_metadata\_mapper.mesoscope package Submodules ---------- -aind\_metadata\_mapper.mesoscope.models module +aind\_metadata\_mapper.mesoscope.mapper module ---------------------------------------------- -.. automodule:: aind_metadata_mapper.mesoscope.models - :members: - :undoc-members: - :show-inheritance: - -aind\_metadata\_mapper.mesoscope.session module ------------------------------------------------ - -.. automodule:: aind_metadata_mapper.mesoscope.session +.. automodule:: aind_metadata_mapper.mesoscope.mapper :members: :undoc-members: :show-inheritance: diff --git a/src/aind_metadata_mapper/mapper_registry.py b/src/aind_metadata_mapper/mapper_registry.py index 92536fa7..6ec2befa 100644 --- a/src/aind_metadata_mapper/mapper_registry.py +++ b/src/aind_metadata_mapper/mapper_registry.py @@ -4,7 +4,9 @@ """ from aind_metadata_mapper.fip.mapper import FIPMapper +from aind_metadata_mapper.mesoscope.mapper import MesoscopeMapper registry = { "fip": FIPMapper, + "mesoscope": MesoscopeMapper, } diff --git a/src/aind_metadata_mapper/mesoscope/README.md b/src/aind_metadata_mapper/mesoscope/README.md new file mode 100644 index 00000000..8a2b9cf6 --- /dev/null +++ b/src/aind_metadata_mapper/mesoscope/README.md @@ -0,0 +1,62 @@ +# Mesoscope mapper + +This mapper transforms extractor-produced `mesoscope.json` into +`acquisition_mesoscope.json` for `GatherMetadataJob`. + +## Input contract + +The mapper consumes the `aind-metadata-extractor` `MesoscopeExtractModel`, +including `tiff_header`, `session_metadata`, `camstim_epchs`, +`camstim_session_type`, and `job_settings`. + +`job_settings.instrument_id` is required. Preparation should populate this +field from the raw `rig.json` `rig_id` when that full instrument identifier is +available. The session or platform alias such as `MESO.2` must not be promoted +to `Acquisition.instrument_id`. + +## Current mapping + +The mapper emits one `ImagingConfig` for the mesoscope stream. It uses the +explicit `job_settings.instrument_id` value for `Acquisition.instrument_id` and +preserves the platform or session alias such as `MESO.2` for +`ImagingConfig.device_name`, matching current production upgrade outputs. + +Plane power preserves the legacy percent values. Calculated milliwatt values +are not mapped. + +Naive timestamps are interpreted in `America/Los_Angeles`, while aware timestamps +are preserved. Acquisition bounds span the recorded stream and stimulus epochs +without altering their individual timestamps. + +The mapper writes the acquisition coordinate system through the +`coordinate_system` field so that it remains compatible with the declared +`aind-data-schema>=2.7.1,<3` dependency floor. Version 2.7.1 does not expose +`global_coordinate_system`. + +Behavior, Eye, and Face camera names are normalized to upgrader-style assembly +names such as `Behavior camera assembly`. + +Missing `stimulus_modalities` are normalized to `Visual`, and omitted +visual-stimulation parameter defaults (`stimulus_type` and `notes`) are filled +from the historical mesoscope Camstim producer contract. This keeps native +mapper output aligned with legacy and upgraded production metadata without +overwriting any values that are already supplied by extraction. + +## Inherited limits + +The nested `Unknown Detector`, `Laser`, and `Ophys Channel` values are +deliberate scientific compatibility defaults inherited from the legacy mapper +and upgrader outputs. Detector and channel-index inference are not included. + +## Sources + +The implementation follows these sources: + +- Legacy mapper: + `v0.29.3:src/aind_metadata_mapper/mesoscope/session.py` +- Legacy Camstim producer contract: + `v0.29.3:src/aind_metadata_mapper/stimulus/camstim.py` +- Released upgrader precedent: + `v0.17.12:src/aind_metadata_upgrader/session/v1v2.py` +- Extractor contract: + `aind_metadata_extractor/models/mesoscope.py` diff --git a/src/aind_metadata_mapper/mesoscope/__init__.py b/src/aind_metadata_mapper/mesoscope/__init__.py new file mode 100644 index 00000000..747ae699 --- /dev/null +++ b/src/aind_metadata_mapper/mesoscope/__init__.py @@ -0,0 +1,5 @@ +"""Mesoscope mapper module.""" + +from aind_metadata_mapper.mesoscope.mapper import MesoscopeMapper + +__all__ = ["MesoscopeMapper"] diff --git a/src/aind_metadata_mapper/mesoscope/mapper.py b/src/aind_metadata_mapper/mesoscope/mapper.py new file mode 100644 index 00000000..2a06ed86 --- /dev/null +++ b/src/aind_metadata_mapper/mesoscope/mapper.py @@ -0,0 +1,600 @@ +"""Mesoscope mapper module.""" + +import json +import re +from datetime import datetime, tzinfo +from typing import Any, Optional +from zoneinfo import ZoneInfo + +from aind_data_schema.components.configs import ( + Channel, + CoupledPlane, + DetectorConfig, + ImagingConfig, + LaserConfig, + PlanarImage, + Plane, + SamplingStrategy, + TriggerType, +) +from aind_data_schema.components.coordinates import CoordinateSystemLibrary, Scale, Translation +from aind_data_schema.components.identifiers import Code, Software +from aind_data_schema.core.acquisition import ( + Acquisition, + AcquisitionSubjectDetails, + DataStream, + PerformanceMetrics, + StimulusEpoch, +) +from aind_data_schema_models.brain_atlas import CCFv3 +from aind_data_schema_models.modalities import Modality +from aind_data_schema_models.stimulus_modality import StimulusModality +from aind_data_schema_models.units import PowerUnit, SizeUnit, VolumeUnit +from aind_metadata_extractor.models.mesoscope import MesoscopeExtractModel + +from aind_metadata_mapper.base import MapperJob, MapperJobSettings + +PACIFIC_TIMEZONE = ZoneInfo("America/Los_Angeles") +DEFAULT_FOV_COORDINATE_UM = 1.5 +DEFAULT_PIXEL_SIZE_UM = 0.78 +DEFAULT_LASER_WAVELENGTH_NM = 920 +DEFAULT_CHANNEL_NAME = "Ophys Channel" +DEFAULT_DETECTOR_NAME = "Unknown Detector" +DEFAULT_LIGHT_SOURCE_NAME = "Laser" +DEFAULT_IMAGE_DEVICE_NAME = "Mesoscope" +DEFAULT_EXPOSURE_TIME_MS = 1.0 +CAMERA_ASSEMBLY_ORDER = {"Behavior": 0, "Eye": 1, "Face": 2} +DEFAULT_VISUAL_STIMULUS_MODALITIES = [StimulusModality.VISUAL] +DEFAULT_VISUAL_STIMULUS_TYPE = "Visual Stimulation" +LEGACY_TARGETED_STRUCTURE_LOOKUP = { + 385: CCFv3.VISP, + 394: CCFv3.VISAM, + 402: CCFv3.VISAL, + 409: CCFv3.VISL, + 417: CCFv3.VISRL, + 533: CCFv3.VISPM, + 312782574: CCFv3.VISLI, +} + + +class MesoscopeMapper(MapperJob): + """Transform extracted mesoscope metadata into Acquisition metadata.""" + + def _validate_metadata(self, metadata: dict) -> MesoscopeExtractModel: + """Validate extracted metadata against MesoscopeExtractModel.""" + return MesoscopeExtractModel.model_validate(metadata) + + def _get_required_value(self, container: dict, key: str, context: str) -> Any: + """Return a required value from a dictionary.""" + value = container.get(key) + if value in (None, ""): + raise ValueError(f"Mesoscope {context}.{key} is required.") + return value + + def _parse_datetime( + self, + value: Any, + field_name: str, + fallback_tz: Optional[tzinfo] = None, + ) -> datetime: + """Parse a datetime and attach Pacific time to naive values.""" + if value is None: + raise ValueError(f"{field_name} is required.") + + if isinstance(value, datetime): + parsed = value + elif isinstance(value, str): + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + else: + raise TypeError(f"{field_name} must be an ISO 8601 string or datetime.") + + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=fallback_tz or PACIFIC_TIMEZONE) + return parsed + + def _parse_float(self, value: Any, field_name: str) -> float: + """Parse a required numeric field.""" + if value in (None, ""): + raise ValueError(f"{field_name} is required.") + return float(value) + + def _parse_optional_float(self, value: Any) -> Optional[float]: + """Parse an optional numeric field.""" + if value in (None, ""): + return None + return float(value) + + def _normalize_camera_base_name(self, name: str) -> str: + """Normalize a camera name to its shared base.""" + normalized = re.sub(r"\s+camera\s+assembly\s*$", "", name, flags=re.IGNORECASE).strip() + normalized = re.sub(r"\s+camera\s*$", "", normalized, flags=re.IGNORECASE).strip() + return normalized + + def _normalize_camera_assembly_name(self, name: str) -> str: + """Normalize a camera name to upgrader-compatible assembly naming.""" + base = self._normalize_camera_base_name(name) + if not base: + raise ValueError("Camera name cannot be empty.") + return f"{base} camera assembly" + + def _extract_camera_label(self, session_key: str, camera_metadata: dict) -> Optional[str]: + """Extract a camera label from extractor session metadata.""" + report = camera_metadata.get("RecordingReport", {}) + label = report.get("CameraLabel") + if isinstance(label, str) and label.strip(): + return label.strip() + + for candidate in CAMERA_ASSEMBLY_ORDER: + if candidate.lower() in session_key.lower(): + return candidate + return None + + def _collect_camera_assembly_names(self, session_metadata: dict) -> list[str]: + """Collect active camera assemblies from session metadata.""" + assemblies = [] + for session_key, camera_metadata in session_metadata.items(): + if session_key == "platform" or not isinstance(camera_metadata, dict): + continue + label = self._extract_camera_label(session_key, camera_metadata) + if label is None: + continue + assemblies.append(self._normalize_camera_assembly_name(label)) + + unique_assemblies = [] + for name in sorted( + assemblies, key=lambda item: CAMERA_ASSEMBLY_ORDER.get(self._normalize_camera_base_name(item), 99) + ): + if name not in unique_assemblies: + unique_assemblies.append(name) + return unique_assemblies + + def _resolve_subject_id(self, platform: dict, job_settings: dict) -> str: + """Resolve and validate subject_id.""" + platform_subject_id = platform.get("subject_id") + job_subject_id = job_settings.get("subject_id") + if platform_subject_id and job_subject_id and str(platform_subject_id) != str(job_subject_id): + raise ValueError( + "Mesoscope platform.subject_id does not match job_settings.subject_id: " + f"{platform_subject_id} != {job_subject_id}." + ) + subject_id = platform_subject_id or job_subject_id + if subject_id in (None, ""): + raise ValueError("Mesoscope subject_id is required.") + return str(subject_id) + + def _resolve_ethics_review_id(self, job_settings: dict) -> Optional[list[str]]: + """Normalize IACUC protocol values.""" + protocol = job_settings.get("iacuc_protocol") + if protocol in (None, "", []): + return None + if isinstance(protocol, list): + normalized = [str(item) for item in protocol if item not in (None, "")] + return normalized or None + return [str(protocol)] + + def _resolve_imaging_device_name(self, platform: dict, instrument_id: str) -> str: + """Resolve imaging device name for ImagingConfig.device_name. + + Native mesoscope mapping keeps the platform/session alias here to match + upgraded production outputs. Acquisition.instrument_id still comes from + the explicit full instrument ID in job_settings.instrument_id. + """ + return str(platform.get("rig_id") or instrument_id) + + def _get_targeted_structure(self, value: Any) -> dict: + """Map a legacy targeted structure value to a CCFv3 structure.""" + if isinstance(value, dict): + return value + + if isinstance(value, str): + stripped = value.strip() + if not stripped: + raise ValueError("targeted_structure_id is required.") + if stripped.isdigit(): + value = int(stripped) + else: + acronym = stripped.upper() + if hasattr(CCFv3, acronym): + return getattr(CCFv3, acronym).model_dump() + raise ValueError(f"Unsupported targeted_structure_id: {value}") + + if isinstance(value, int): + if value in LEGACY_TARGETED_STRUCTURE_LOOKUP: + return LEGACY_TARGETED_STRUCTURE_LOOKUP[value].model_dump() + raise ValueError(f"Unsupported targeted_structure_id: {value}") + + raise TypeError(f"Unsupported targeted_structure_id type: {type(value).__name__}") + + def _validate_fov_reference(self, job_settings: dict) -> None: + """Validate that the mapper only emits Bregma-referenced coordinates.""" + reference = job_settings.get("fov_reference") + if reference and str(reference).lower() != "bregma": + raise ValueError(f"Unsupported fov_reference: {reference}") + + def _get_fov_translation_mm(self, job_settings: dict) -> list[float]: + """Return FOV AP/ML translation in millimeters.""" + self._validate_fov_reference(job_settings) + ap_um = float(job_settings.get("fov_coordinate_ap", DEFAULT_FOV_COORDINATE_UM)) + ml_um = float(job_settings.get("fov_coordinate_ml", DEFAULT_FOV_COORDINATE_UM)) + return [ap_um / 1000.0, ml_um / 1000.0, 0.0] + + def _get_image_dimensions(self, tiff_header: list) -> Scale: + """Extract image dimensions from tiff metadata.""" + if not tiff_header or not isinstance(tiff_header[0], dict): + raise ValueError("Mesoscope tiff_header[0] must be present.") + + header = tiff_header[0] + width = self._parse_float( + header.get("SI.hRoiManager.pixelsPerLine"), "tiff_header[0].SI.hRoiManager.pixelsPerLine" + ) + height = self._parse_float( + header.get("SI.hRoiManager.linesPerFrame"), + "tiff_header[0].SI.hRoiManager.linesPerFrame", + ) + return Scale(scale=[width, height]) + + def _get_scale_transform(self) -> Scale: + """Return legacy pixel scale for planar images.""" + scale_mm = DEFAULT_PIXEL_SIZE_UM / 1000.0 + return Scale(scale=[scale_mm, scale_mm]) + + def _build_imaging_channel(self) -> Channel: + """Build default mesoscope imaging channel.""" + return Channel( + channel_name=DEFAULT_CHANNEL_NAME, + detector=DetectorConfig( + device_name=DEFAULT_DETECTOR_NAME, + exposure_time=DEFAULT_EXPOSURE_TIME_MS, + trigger_type=TriggerType.INTERNAL, + ), + light_sources=[ + LaserConfig( + device_name=DEFAULT_LIGHT_SOURCE_NAME, + wavelength=DEFAULT_LASER_WAVELENGTH_NM, + wavelength_unit=SizeUnit.NM, + power=None, + power_unit=PowerUnit.MW, + ) + ], + ) + + def _get_plane_power(self, group: dict, plane: dict, group_index: int, plane_index: int) -> float: + """Resolve plane power from group or plane data.""" + group_power = self._parse_optional_float(group.get("scanimage_power_percent")) + if group_power is not None: + return group_power + + plane_power = self._parse_optional_float(plane.get("scanimage_power")) + if plane_power is not None: + return plane_power + + raise ValueError( + "Mesoscope imaging_plane_groups[" + f"{group_index}] plane[{plane_index}] is missing scanimage_power_percent and scanimage_power." + ) + + def _build_plane( + self, group: dict, plane: dict, group_index: int, plane_index: int, global_index: int + ) -> Plane | CoupledPlane: + """Build a Plane or CoupledPlane from a platform imaging plane.""" + targeted_structure = self._get_targeted_structure(plane.get("targeted_structure_id")) + base_kwargs = dict( + depth=self._parse_float( + plane.get("targeted_depth"), f"imaging_plane_groups[{group_index}].planes[{plane_index}].targeted_depth" + ), + depth_unit=SizeUnit.UM, + power=self._get_plane_power(group, plane, group_index, plane_index), + power_unit=PowerUnit.PERCENT, + targeted_structure=targeted_structure, + ) + + if len(group["imaging_planes"]) == 1: + return Plane(**base_kwargs) + + partner_index = global_index + 1 if plane_index == 0 else global_index - 1 + power_ratio = self._parse_float( + group.get("scanimage_split_percent"), + f"imaging_plane_groups[{group_index}].scanimage_split_percent", + ) + return CoupledPlane( + **base_kwargs, + plane_index=global_index, + coupled_plane_index=partner_index, + power_ratio=power_ratio, + ) + + def _build_stream_notes( + self, plane_index: int, scanimage_roi_index: Any, magnification: Optional[str] + ) -> Optional[str]: + """Build stream notes for plane metadata without dedicated schema fields.""" + extras = [] + if magnification not in (None, ""): + extras.append(f"magnification={magnification}") + if scanimage_roi_index is not None: + extras.append(f"scanimage_roi_index={scanimage_roi_index}") + if not extras: + return None + return f"FOV {plane_index}: " + ", ".join(extras) + + def _build_imaging_config( + self, + tiff_header: list, + platform: dict, + job_settings: dict, + imaging_device_name: str, + ) -> tuple[ImagingConfig, Optional[str]]: + """Build ImagingConfig and any stream notes.""" + dimensions = self._get_image_dimensions(tiff_header) + translation = Translation(translation=self._get_fov_translation_mm(job_settings)) + channel = self._build_imaging_channel() + images = [] + notes = [] + global_plane_index = 0 + magnification = job_settings.get("magnification") + + groups = self._get_required_value(platform, "imaging_plane_groups", "platform") + if not groups: + raise ValueError("Mesoscope platform.imaging_plane_groups must contain at least one group.") + for group_index, group in enumerate(groups): + planes = self._get_required_value(group, "imaging_planes", f"platform.imaging_plane_groups[{group_index}]") + if len(planes) not in (1, 2): + raise ValueError( + f"Mesoscope imaging_plane_groups[{group_index}] must contain 1 or 2 " + f"imaging_planes, got {len(planes)}." + ) + + for plane_index, plane in enumerate(planes): + schema_plane = self._build_plane(group, plane, group_index, plane_index, global_plane_index) + images.append( + PlanarImage( + channel_name=channel.channel_name, + planes=[schema_plane], + image_to_acquisition_transform=[self._get_scale_transform(), translation], + dimensions=dimensions, + dimensions_unit=SizeUnit.PX, + ) + ) + note = self._build_stream_notes(global_plane_index, plane.get("scanimage_roi_index"), magnification) + if note is not None: + notes.append(note) + global_plane_index += 1 + + imaging_config = ImagingConfig( + device_name=imaging_device_name, + channels=[channel], + images=images, + sampling_strategy=SamplingStrategy( + frame_rate=self._parse_float( + groups[0].get("acquisition_framerate_Hz"), + "imaging_plane_groups[0].acquisition_framerate_Hz", + ) + ), + ) + return imaging_config, "; ".join(notes) if notes else None + + def _build_performance_metrics(self, epoch: dict) -> Optional[PerformanceMetrics]: + """Build performance metrics when present.""" + reward_value = epoch.get("reward_consumed_during_epoch") + reward_unit = epoch.get("reward_consumed_unit") + if reward_value not in (None, "") and reward_unit is None: + reward_unit = VolumeUnit.ML if float(reward_value) < 3 else VolumeUnit.UL + + if not any( + [ + epoch.get("trials_total"), + epoch.get("trials_finished"), + epoch.get("trials_rewarded"), + reward_value, + ] + ): + return None + + return PerformanceMetrics( + trials_total=epoch.get("trials_total"), + trials_finished=epoch.get("trials_finished"), + trials_rewarded=epoch.get("trials_rewarded"), + reward_consumed_during_epoch=reward_value, + reward_consumed_unit=reward_unit, + output_parameters=epoch.get("output_parameters"), + ) + + def _get_stimulus_parameters(self, stimulus_parameters: Any) -> Optional[dict]: + """Normalize the extracted stimulus parameter structure.""" + parameters = None + if isinstance(stimulus_parameters, list): + if len(stimulus_parameters) == 1: + if not isinstance(stimulus_parameters[0], dict): + raise TypeError("stimulus_parameters entries must be dictionaries.") + parameters = self._normalize_visual_stimulation_parameters(stimulus_parameters[0]) + else: + parameters = {} + for index, item in enumerate(stimulus_parameters): + if not isinstance(item, dict): + raise TypeError("stimulus_parameters entries must be dictionaries.") + key = item.get("stimulus_name") or f"Stimulus_{index}" + if key in parameters: + key = f"{key}_{index}" + parameters[key] = self._normalize_visual_stimulation_parameters(item) + elif stimulus_parameters is not None: + if not isinstance(stimulus_parameters, dict): + raise TypeError("stimulus_parameters must be a dictionary or list of dictionaries.") + parameters = stimulus_parameters + return parameters + + def _build_code(self, epoch: dict) -> Optional[Code]: + """Build a Code object from extracted Camstim epoch data.""" + script = epoch.get("script") or {} + parameters = self._get_stimulus_parameters(epoch.get("stimulus_parameters")) + software = epoch.get("software") or [] + core_dependency = None + if software: + first_software = software[0] + core_dependency = Software( + name=first_software.get("name", "unknown"), + version=first_software.get("version"), + ) + + if script.get("parameters"): + if parameters is None: + parameters = script["parameters"] + else: + parameters = parameters | script["parameters"] + + if not script and parameters is None and core_dependency is None: + return None + + if not script: + return Code( + name="Code", + version="unknown", + url="unknown", + parameters=parameters, + core_dependency=core_dependency, + ) + + return Code( + name=script.get("name", "Unknown Script"), + version=script.get("version", "unknown"), + url=script.get("url", "") or "", + parameters=parameters, + core_dependency=core_dependency, + ) + + def _resolve_stimulus_modalities(self, epoch: dict) -> list[StimulusModality]: + """Resolve stimulus modalities from extractor output.""" + modalities = epoch.get("stimulus_modalities") + if modalities: + return modalities + return DEFAULT_VISUAL_STIMULUS_MODALITIES + + def _normalize_visual_stimulation_parameters(self, parameters: dict[str, Any]) -> dict[str, Any]: + """Add legacy visual-stimulation defaults without overwriting supplied values.""" + normalized = dict(parameters) + normalized.setdefault("stimulus_type", DEFAULT_VISUAL_STIMULUS_TYPE) + normalized.setdefault("notes", None) + return normalized + + def _build_stimulus_epoch(self, epoch: dict, fallback_tz: tzinfo) -> StimulusEpoch: + """Build a StimulusEpoch from extracted Camstim epoch data.""" + task_parameters = (epoch.get("output_parameters") or {}).get("task_parameters") or {} + stimulus_start = self._parse_datetime(epoch.get("stimulus_start_time"), "stimulus_start_time", fallback_tz) + stimulus_end = self._parse_datetime(epoch.get("stimulus_end_time"), "stimulus_end_time", fallback_tz) + if stimulus_end < stimulus_start: + raise ValueError("stimulus_end_time must be greater than or equal to stimulus_start_time.") + + return StimulusEpoch( + stimulus_start_time=stimulus_start, + stimulus_end_time=stimulus_end, + stimulus_name=self._get_required_value(epoch, "stimulus_name", "camstim_epchs"), + stimulus_modalities=self._resolve_stimulus_modalities(epoch), + performance_metrics=self._build_performance_metrics(epoch), + code=self._build_code(epoch), + active_devices=epoch.get("stimulus_device_names") or [], + notes=epoch.get("notes"), + curriculum_status=task_parameters.get("stage_in_use"), + ) + + def _get_acquisition_bounds( + self, + stream_start: datetime, + stream_end: datetime, + stimulus_epochs: list[StimulusEpoch], + ) -> tuple[datetime, datetime]: + """Return acquisition bounds spanning stream and stimulus intervals.""" + acquisition_start = stream_start + acquisition_end = stream_end + + for epoch in stimulus_epochs: + acquisition_start = min(acquisition_start, epoch.stimulus_start_time) + acquisition_end = max(acquisition_end, epoch.stimulus_end_time) + + return acquisition_start, acquisition_end + + def transform(self, metadata: dict) -> Acquisition: + """Transform extracted mesoscope metadata into Acquisition.""" + metadata_model = self._validate_metadata(metadata) + raw_metadata = metadata_model.model_dump() + + tiff_header = self._get_required_value(raw_metadata, "tiff_header", "metadata") + session_metadata = self._get_required_value(raw_metadata, "session_metadata", "metadata") + platform = self._get_required_value(session_metadata, "platform", "session_metadata") + job_settings = self._get_required_value(raw_metadata, "job_settings", "metadata") + instrument_id = str(self._get_required_value(job_settings, "instrument_id", "job_settings")) + acquisition_type = raw_metadata.get("camstim_session_type") or platform.get("stimulus_name") + if acquisition_type in (None, ""): + raise ValueError("Mesoscope camstim_session_type is required.") + + subject_id = self._resolve_subject_id(platform, job_settings) + session_start = self._parse_datetime( + self._get_required_value(job_settings, "session_start_time", "job_settings"), + "session_start_time", + ) + session_end = self._parse_datetime( + self._get_required_value(job_settings, "session_end_time", "job_settings"), + "session_end_time", + session_start.tzinfo, + ) + if session_end < session_start: + raise ValueError("session_end_time must be greater than or equal to session_start_time.") + + imaging_device_name = self._resolve_imaging_device_name(platform, instrument_id) + imaging_config, stream_notes = self._build_imaging_config( + tiff_header, platform, job_settings, imaging_device_name + ) + + active_devices = [imaging_device_name, DEFAULT_IMAGE_DEVICE_NAME] + active_devices.extend(self._collect_camera_assembly_names(session_metadata)) + + data_stream = DataStream( + stream_start_time=session_start, + stream_end_time=session_end, + modalities=[Modality.POPHYS], + active_devices=list(dict.fromkeys(active_devices)), + configurations=[imaging_config], + connections=[], + notes=stream_notes, + ) + + stimulus_epochs = [ + self._build_stimulus_epoch(epoch, session_start.tzinfo or PACIFIC_TIMEZONE) + for epoch in raw_metadata.get("camstim_epchs", []) + ] + acquisition_start, acquisition_end = self._get_acquisition_bounds(session_start, session_end, stimulus_epochs) + + subject_details = None + if job_settings.get("mouse_platform_name"): + subject_details = AcquisitionSubjectDetails( + mouse_platform_name=str(job_settings["mouse_platform_name"]), + reward_consumed_unit=VolumeUnit.ML, + ) + + experimenters = job_settings.get("experimenter_full_name", []) + if isinstance(experimenters, str): + experimenters = [experimenters] + + return Acquisition( + subject_id=subject_id, + acquisition_start_time=acquisition_start, + acquisition_end_time=acquisition_end, + experimenters=[name for name in experimenters if name], + ethics_review_id=self._resolve_ethics_review_id(job_settings), + instrument_id=instrument_id, + acquisition_type=str(acquisition_type), + # aind-data-schema 2.7.1 does not expose global_coordinate_system. + # Use coordinate_system to preserve the declared schema floor. + coordinate_system=CoordinateSystemLibrary.BREGMA_ARI, + data_streams=[data_stream], + stimulus_epochs=stimulus_epochs, + subject_details=subject_details, + ) + + def run_job(self, job_settings: MapperJobSettings) -> None: + """Run mesoscope mapper and write acquisition metadata.""" + with open(job_settings.input_filepath, "r", encoding="utf-8") as input_stream: + metadata = json.load(input_stream) + + acquisition = self.transform(metadata) + acquisition.write_standard_file( + output_directory=job_settings.output_directory, + filename_suffix=job_settings.output_filename_suffix, + ) diff --git a/tests/fixtures/mesoscope_intermediate.json b/tests/fixtures/mesoscope_intermediate.json new file mode 100644 index 00000000..feac53d4 --- /dev/null +++ b/tests/fixtures/mesoscope_intermediate.json @@ -0,0 +1,118 @@ +{ + "tiff_header": [ + { + "SI.hRoiManager.pixelsPerLine": 512, + "SI.hRoiManager.linesPerFrame": 512 + } + ], + "session_metadata": { + "platform": { + "rig_id": "MESO.2", + "session_id": 1770928800, + "subject_id": "825546", + "stimulus_name": "STAGE_1", + "imaging_plane_groups": [ + { + "local_z_stack_tif": "1770928800_local_z_stack0.tiff", + "acquisition_framerate_Hz": 42.85, + "scanimage_power_percent": 88, + "scanimage_split_percent": 28, + "imaging_planes": [ + { + "targeted_depth": 175, + "targeted_structure_id": "VISp", + "scanimage_roi_index": 0, + "scanimage_scanfield_z": -185, + "channel": "Channel2", + "registration": { + "pixel_size_um": 0.78 + } + }, + { + "targeted_depth": 250, + "targeted_structure_id": "VISp", + "scanimage_roi_index": 0, + "scanimage_scanfield_z": -95, + "channel": "Channel1", + "registration": { + "pixel_size_um": 0.78 + } + } + ] + } + ] + }, + "1770928800_Behavior_20260212T130408": { + "RecordingReport": { + "CameraLabel": "Behavior", + "TimeStart": "2026-02-12T13:04:08Z", + "TimeEnd": "2026-02-12T14:07:24Z" + } + }, + "1770928800_Eye_20260212T130408": { + "RecordingReport": { + "CameraLabel": "Eye", + "TimeStart": "2026-02-12T13:04:09Z", + "TimeEnd": "2026-02-12T14:07:24Z" + } + }, + "1770928800_Face_20260212T130409": { + "RecordingReport": { + "CameraLabel": "Face", + "TimeStart": "2026-02-12T13:04:09Z", + "TimeEnd": "2026-02-12T14:07:24Z" + } + } + }, + "camstim_epchs": [ + { + "stimulus_start_time": "2026-02-12T13:05:03.110704-08:00", + "stimulus_end_time": "2026-02-12T13:19:02.807314-08:00", + "stimulus_name": "drifting_gratings_contrast", + "stimulus_parameters": [ + { + "stimulus_name": "drifting_gratings_contrast", + "stimulus_template_name": [], + "stimulus_parameters": { + "contrast": [ + 0.2, + 0.8 + ], + "orientation": [ + 0, + 90 + ] + } + } + ], + "script": { + "name": "STAGE_1", + "version": "1.0" + }, + "software": [ + { + "name": "camstim", + "url": "https://eng-gitlab.corp.alleninstitute.org/braintv/camstim", + "version": "1.0" + } + ] + } + ], + "camstim_session_type": "STAGE_1", + "job_settings": { + "session_id": "1770928800", + "subject_id": "825546", + "instrument_id": "422_MESO2_20260122", + "session_start_time": "2026-02-12T13:04:36.555247-08:00", + "session_end_time": "2026-02-12T14:07:12.672057-08:00", + "experimenter_full_name": [ + "Sam Seid" + ], + "iacuc_protocol": "2115", + "mouse_platform_name": "disc", + "magnification": "16x", + "fov_coordinate_ml": 1.5, + "fov_coordinate_ap": 1.5, + "fov_reference": "Bregma" + } +} diff --git a/tests/mesoscope/test_mapper.py b/tests/mesoscope/test_mapper.py new file mode 100644 index 00000000..2cdd3a71 --- /dev/null +++ b/tests/mesoscope/test_mapper.py @@ -0,0 +1,541 @@ +"""Tests for mesoscope mapper.""" + +import copy +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch +from zoneinfo import ZoneInfo + +from aind_data_schema.core.acquisition import Acquisition +from aind_data_schema_models.modalities import Modality +from aind_data_schema_models.stimulus_modality import StimulusModality + +from aind_metadata_mapper.base import MapperJobSettings +from aind_metadata_mapper.gather_metadata import GatherMetadataJob +from aind_metadata_mapper.mapper_registry import registry +from aind_metadata_mapper.mesoscope.mapper import MesoscopeMapper +from aind_metadata_mapper.models import DataDescriptionSettings, JobSettings + +TEST_DIR = Path(__file__).resolve().parents[1] +FIXTURE_PATH = TEST_DIR / "fixtures" / "mesoscope_intermediate.json" + + +class TestMesoscopeMapper(unittest.TestCase): + """Unit tests for MesoscopeMapper.""" + + def setUp(self) -> None: + """Load fixture data.""" + self.mapper = MesoscopeMapper() + with open(FIXTURE_PATH, "r", encoding="utf-8") as fixture_stream: + self.fixture = json.load(fixture_stream) + + def test_registry_registration(self) -> None: + """Mesoscope mapper should be registered for GatherMetadataJob.""" + self.assertIs(registry["mesoscope"], MesoscopeMapper) + + def test_transform_maps_two_coupled_planes(self) -> None: + """Mapper should convert two-plane mesoscope data into Acquisition.""" + acquisition = self.mapper.transform(copy.deepcopy(self.fixture)) + + self.assertEqual(acquisition.subject_id, "825546") + self.assertEqual(acquisition.instrument_id, "422_MESO2_20260122") + self.assertEqual(acquisition.acquisition_type, "STAGE_1") + self.assertEqual(acquisition.ethics_review_id, ["2115"]) + self.assertEqual(acquisition.model_dump(mode="json")["coordinate_system"]["name"], "BREGMA_ARI") + self.assertEqual(acquisition.subject_details.mouse_platform_name, "disc") + self.assertEqual(acquisition.subject_details.reward_consumed_unit, "milliliter") + + stream = acquisition.data_streams[0] + self.assertEqual(stream.active_devices[0], "MESO.2") + self.assertIn("Mesoscope", stream.active_devices) + self.assertIn("Behavior camera assembly", stream.active_devices) + self.assertIn("Eye camera assembly", stream.active_devices) + self.assertIn("Face camera assembly", stream.active_devices) + self.assertIn("magnification=16x", stream.notes) + self.assertIn("scanimage_roi_index=0", stream.notes) + + imaging_config = stream.configurations[0] + self.assertEqual(imaging_config.device_name, "MESO.2") + self.assertEqual(imaging_config.channels[0].channel_name, "Ophys Channel") + self.assertEqual(imaging_config.channels[0].detector.device_name, "Unknown Detector") + self.assertEqual(imaging_config.channels[0].light_sources[0].device_name, "Laser") + self.assertEqual(imaging_config.channels[0].light_sources[0].wavelength, 920) + + self.assertEqual(len(imaging_config.images), 2) + first_plane = imaging_config.images[0].planes[0] + second_plane = imaging_config.images[1].planes[0] + self.assertEqual(first_plane.plane_index, 0) + self.assertEqual(first_plane.coupled_plane_index, 1) + self.assertEqual(second_plane.plane_index, 1) + self.assertEqual(second_plane.coupled_plane_index, 0) + self.assertEqual(first_plane.power, 88.0) + self.assertEqual(first_plane.power_ratio, 28.0) + self.assertEqual(first_plane.targeted_structure.acronym, "VISp") + + first_transform = imaging_config.images[0].image_to_acquisition_transform + self.assertEqual(first_transform[0].scale, [0.00078, 0.00078]) + self.assertEqual(first_transform[1].translation, [0.0015, 0.0015, 0.0]) + self.assertEqual(imaging_config.images[0].dimensions.scale, [512.0, 512.0]) + + stimulus_epoch = acquisition.stimulus_epochs[0] + self.assertEqual(stimulus_epoch.stimulus_name, "drifting_gratings_contrast") + self.assertEqual(stimulus_epoch.code.name, "STAGE_1") + self.assertEqual(stimulus_epoch.code.url, "") + self.assertEqual(stimulus_epoch.code.core_dependency.name, "camstim") + stimulus_parameters = stimulus_epoch.code.parameters.model_dump() + self.assertEqual(stimulus_epoch.stimulus_modalities, [StimulusModality.VISUAL]) + self.assertEqual(stimulus_parameters["stimulus_type"], "Visual Stimulation") + self.assertIsNone(stimulus_parameters["notes"]) + self.assertEqual( + stimulus_parameters["stimulus_parameters"]["contrast"], + [0.2, 0.8], + ) + + def test_transform_maps_multiple_groups_in_platform_order(self) -> None: + """Mapper should keep platform plane order across multiple groups.""" + metadata = copy.deepcopy(self.fixture) + metadata["session_metadata"]["platform"]["imaging_plane_groups"].append( + { + "local_z_stack_tif": "1770928800_local_z_stack1.tiff", + "acquisition_framerate_Hz": 42.85, + "scanimage_power_percent": 90, + "scanimage_split_percent": 13, + "imaging_planes": [ + { + "targeted_depth": 325, + "targeted_structure_id": "VISam", + "scanimage_roi_index": 1, + "registration": {"pixel_size_um": 0.78}, + }, + { + "targeted_depth": 400, + "targeted_structure_id": "VISpm", + "scanimage_roi_index": 1, + "registration": {"pixel_size_um": 0.78}, + }, + ], + } + ) + + acquisition = self.mapper.transform(metadata) + imaging_config = acquisition.data_streams[0].configurations[0] + + self.assertEqual(len(imaging_config.images), 4) + plane_indices = [image.planes[0].plane_index for image in imaging_config.images] + coupled_indices = [image.planes[0].coupled_plane_index for image in imaging_config.images] + depths = [image.planes[0].depth for image in imaging_config.images] + + self.assertEqual(plane_indices, [0, 1, 2, 3]) + self.assertEqual(coupled_indices, [1, 0, 3, 2]) + self.assertEqual(depths, [175.0, 250.0, 325.0, 400.0]) + self.assertEqual(imaging_config.images[2].planes[0].power_ratio, 13.0) + + def test_transform_uses_plane_power_when_group_power_missing(self) -> None: + """Mapper should fall back to plane-level percent power.""" + metadata = copy.deepcopy(self.fixture) + group = metadata["session_metadata"]["platform"]["imaging_plane_groups"][0] + group.pop("scanimage_power_percent") + group["imaging_planes"][0]["scanimage_power"] = 61 + group["imaging_planes"][1]["scanimage_power"] = 62 + + acquisition = self.mapper.transform(metadata) + imaging_config = acquisition.data_streams[0].configurations[0] + + self.assertEqual(imaging_config.images[0].planes[0].power, 61.0) + self.assertEqual(imaging_config.images[1].planes[0].power, 62.0) + + def test_transform_assigns_pacific_timezone_to_naive_times(self) -> None: + """Naive timestamps should use America/Los_Angeles.""" + metadata = copy.deepcopy(self.fixture) + metadata["job_settings"]["session_start_time"] = "2026-02-12T13:04:36.555247" + metadata["job_settings"]["session_end_time"] = "2026-02-12T14:07:12.672057" + metadata["camstim_epchs"][0]["stimulus_start_time"] = "2026-02-12T13:05:03.110704" + metadata["camstim_epchs"][0]["stimulus_end_time"] = "2026-02-12T13:19:02.807314" + + acquisition = self.mapper.transform(metadata) + + self.assertEqual(str(acquisition.acquisition_start_time.tzinfo), "America/Los_Angeles") + self.assertEqual(str(acquisition.stimulus_epochs[0].stimulus_start_time.tzinfo), "America/Los_Angeles") + + def test_transform_raises_on_missing_instrument_id(self) -> None: + """Mapper should fail without explicit job_settings.instrument_id.""" + metadata = copy.deepcopy(self.fixture) + metadata["job_settings"].pop("instrument_id") + + with self.assertRaises(ValueError) as context: + self.mapper.transform(metadata) + + self.assertIn("job_settings.instrument_id", str(context.exception)) + + def test_transform_raises_on_missing_acquisition_type(self) -> None: + """Mapper should fail without a session type.""" + metadata = copy.deepcopy(self.fixture) + metadata["camstim_session_type"] = "" + metadata["session_metadata"]["platform"]["stimulus_name"] = "" + + with self.assertRaises(ValueError) as context: + self.mapper.transform(metadata) + + self.assertIn("camstim_session_type", str(context.exception)) + + def test_transform_raises_when_session_times_are_inverted(self) -> None: + """Mapper should fail when session end is before start.""" + metadata = copy.deepcopy(self.fixture) + metadata["job_settings"]["session_start_time"] = "2026-02-12T14:07:12.672057-08:00" + metadata["job_settings"]["session_end_time"] = "2026-02-12T13:04:36.555247-08:00" + + with self.assertRaises(ValueError) as context: + self.mapper.transform(metadata) + + self.assertIn("session_end_time", str(context.exception)) + + def test_transform_accepts_string_experimenter(self) -> None: + """Mapper should normalize a single experimenter string.""" + metadata = copy.deepcopy(self.fixture) + metadata["job_settings"]["experimenter_full_name"] = "Sam Seid" + + acquisition = self.mapper.transform(metadata) + + self.assertEqual(acquisition.experimenters, ["Sam Seid"]) + + def test_transform_raises_on_malformed_pairing(self) -> None: + """Mapper should fail when a plane group has more than two planes.""" + metadata = copy.deepcopy(self.fixture) + metadata["session_metadata"]["platform"]["imaging_plane_groups"][0]["imaging_planes"].append( + { + "targeted_depth": 300, + "targeted_structure_id": "VISp", + "scanimage_roi_index": 0, + "registration": {"pixel_size_um": 0.78}, + "scanimage_power": 70, + } + ) + + with self.assertRaises(ValueError) as context: + self.mapper.transform(metadata) + + self.assertIn("must contain 1 or 2 imaging_planes", str(context.exception)) + + def test_transform_extends_acquisition_to_stimulus_interval(self) -> None: + """Acquisition bounds should span stream and stimulus times.""" + metadata = copy.deepcopy(self.fixture) + metadata["job_settings"]["session_end_time"] = "2026-02-20T11:17:49.592634-08:00" + metadata["camstim_epchs"][0]["stimulus_start_time"] = "2026-02-20T10:07:44.569904-08:00" + metadata["camstim_epchs"][0]["stimulus_end_time"] = "2026-02-20T11:17:55.926134-08:00" + + acquisition = self.mapper.transform(metadata) + + self.assertEqual(acquisition.acquisition_end_time.isoformat(), "2026-02-20T11:17:55.926134-08:00") + self.assertEqual( + acquisition.data_streams[0].stream_end_time.isoformat(), + "2026-02-20T11:17:49.592634-08:00", + ) + self.assertEqual( + acquisition.stimulus_epochs[0].stimulus_end_time.isoformat(), + "2026-02-20T11:17:55.926134-08:00", + ) + + def test_transform_rejects_inverted_stimulus_interval(self) -> None: + """Mapper should reject inverted stimulus intervals.""" + metadata = copy.deepcopy(self.fixture) + metadata["camstim_epchs"][0]["stimulus_start_time"] = "2026-02-12T13:19:02.807314-08:00" + metadata["camstim_epchs"][0]["stimulus_end_time"] = "2026-02-12T13:05:03.110704-08:00" + + with self.assertRaises(ValueError) as context: + self.mapper.transform(metadata) + + self.assertIn("stimulus_end_time", str(context.exception)) + + def test_transform_defaults_missing_stimulus_modalities_to_visual(self) -> None: + """Mapper should normalize missing mesoscope Camstim modalities to Visual.""" + metadata = copy.deepcopy(self.fixture) + metadata["camstim_epchs"][0].pop("stimulus_modalities", None) + + acquisition = self.mapper.transform(metadata) + + self.assertEqual(acquisition.stimulus_epochs[0].stimulus_modalities, [StimulusModality.VISUAL]) + + def test_transform_preserves_explicit_stimulus_modalities(self) -> None: + """Mapper should preserve explicit upstream stimulus fields.""" + metadata = copy.deepcopy(self.fixture) + metadata["camstim_epchs"][0]["stimulus_modalities"] = [StimulusModality.VISUAL] + metadata["camstim_epchs"][0]["stimulus_parameters"][0]["stimulus_type"] = "Custom visual" + metadata["camstim_epchs"][0]["stimulus_parameters"][0]["notes"] = "keep me" + + acquisition = self.mapper.transform(metadata) + + self.assertEqual(acquisition.stimulus_epochs[0].stimulus_modalities, [StimulusModality.VISUAL]) + parameters = acquisition.stimulus_epochs[0].code.parameters.model_dump() + self.assertEqual(parameters["stimulus_type"], "Custom visual") + self.assertEqual(parameters["notes"], "keep me") + + def test_transform_ignores_registration_pixel_size(self) -> None: + """Baseline mapper should keep legacy default pixel scale.""" + metadata = copy.deepcopy(self.fixture) + metadata["session_metadata"]["platform"]["imaging_plane_groups"][0]["imaging_planes"][0]["registration"][ + "pixel_size_um" + ] = 0.4962 + + acquisition = self.mapper.transform(metadata) + + scale = acquisition.data_streams[0].configurations[0].images[0].image_to_acquisition_transform[0].scale + self.assertEqual(scale, [0.00078, 0.00078]) + + def test_run_job_writes_acquisition_mesoscope_json(self) -> None: + """run_job should write standard acquisition output.""" + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) + input_path = runtime_dir / "mesoscope.json" + with open(input_path, "w", encoding="utf-8") as output_stream: + json.dump(self.fixture, output_stream) + + settings = MapperJobSettings( + input_filepath=input_path, + output_directory=runtime_dir, + output_filename_suffix="mesoscope", + ) + self.mapper.run_job(settings) + + output_path = runtime_dir / "acquisition_mesoscope.json" + self.assertTrue(output_path.exists()) + + with open(output_path, "r", encoding="utf-8") as input_stream: + acquisition = Acquisition.model_validate(json.load(input_stream)) + self.assertEqual(acquisition.instrument_id, "422_MESO2_20260122") + + def test_gather_metadata_job_runs_registered_mapper_without_network(self) -> None: + """GatherMetadataJob should discover and run mesoscope mapper.""" + with tempfile.TemporaryDirectory() as tmpdir: + runtime_dir = Path(tmpdir) + with open(runtime_dir / "mesoscope.json", "w", encoding="utf-8") as output_stream: + json.dump(self.fixture, output_stream) + + job = GatherMetadataJob( + settings=JobSettings( + metadata_dir=str(runtime_dir), + output_dir=str(runtime_dir), + subject_id="825546", + data_description_settings=DataDescriptionSettings( + project_name="Mesoscope test", + modalities=[Modality.POPHYS], + ), + ) + ) + + with ( + patch.object(job, "copy_original_metadata_files"), + patch.object( + job, + "build_data_description", + return_value={ + "name": "multiplane-ophys_825546_2026-02-12_13-04-36", + "object_type": "Data description", + }, + ), + patch.object(job, "get_subject", return_value=None), + patch.object(job, "get_procedures", return_value=None), + patch.object(job, "get_instrument", return_value=None), + patch.object(job, "get_processing", return_value=None), + patch.object(job, "get_quality_control", return_value=None), + patch.object(job, "get_model", return_value=None), + patch.object(job, "validate_and_create_metadata", return_value={}), + ): + job.run_job() + + self.assertTrue((runtime_dir / "acquisition_mesoscope.json").exists()) + self.assertTrue((runtime_dir / "acquisition.json").exists()) + with open(runtime_dir / "acquisition.json", "r", encoding="utf-8") as input_stream: + acquisition = Acquisition.model_validate(json.load(input_stream)) + self.assertEqual(acquisition.data_streams[0].configurations[0].device_name, "MESO.2") + self.assertIn("Behavior camera assembly", acquisition.data_streams[0].active_devices) + + def test_parse_datetime_helper(self) -> None: + """Datetime helper should preserve aware values and reject bad input.""" + aware_value = self.mapper._parse_datetime("2026-02-12T13:04:36.555247-08:00", "field") + self.assertEqual(aware_value.isoformat(), "2026-02-12T13:04:36.555247-08:00") + + from datetime import datetime + + now = datetime(2026, 2, 12, 13, 4, 36, tzinfo=ZoneInfo("UTC")) + self.assertIs(self.mapper._parse_datetime(now, "field"), now) + + with self.assertRaises(ValueError): + self.mapper._parse_datetime(None, "field") + with self.assertRaises(TypeError): + self.mapper._parse_datetime(5, "field") + + def test_parse_float_missing_raises(self) -> None: + """Numeric helper should require values.""" + with self.assertRaises(ValueError): + self.mapper._parse_float("", "field") + + def test_camera_name_helpers(self) -> None: + """Camera helpers should normalize fallback and unlabeled inputs.""" + self.assertEqual( + self.mapper._extract_camera_label("1770928800_Eye_20260212T130408", {"RecordingReport": {}}), + "Eye", + ) + self.assertIsNone(self.mapper._extract_camera_label("unknown_key", {"RecordingReport": {}})) + + session_metadata = { + "platform": {}, + "camera_a": {"RecordingReport": {"CameraLabel": "Face Camera"}}, + "camera_b": {"RecordingReport": {}}, + } + self.assertEqual( + self.mapper._collect_camera_assembly_names(session_metadata), + ["Face camera assembly"], + ) + + with self.assertRaises(ValueError): + self.mapper._normalize_camera_assembly_name(" ") + + def test_subject_and_ethics_helpers(self) -> None: + """Subject and ethics helpers should validate mismatches and empty values.""" + with self.assertRaises(ValueError): + self.mapper._resolve_subject_id({"subject_id": "1"}, {"subject_id": "2"}) + with self.assertRaises(ValueError): + self.mapper._resolve_subject_id({}, {}) + + self.assertIsNone(self.mapper._resolve_ethics_review_id({})) + self.assertIsNone(self.mapper._resolve_ethics_review_id({"iacuc_protocol": [""]})) + self.assertEqual( + self.mapper._resolve_ethics_review_id({"iacuc_protocol": ["2115", "2414"]}), + ["2115", "2414"], + ) + + def test_targeted_structure_helper(self) -> None: + """Targeted structure helper should cover supported and invalid values.""" + structure_dict = {"acronym": "VISp"} + self.assertEqual(self.mapper._get_targeted_structure(structure_dict), structure_dict) + self.assertEqual(self.mapper._get_targeted_structure("385")["acronym"], "VISp") + + with self.assertRaises(ValueError): + self.mapper._get_targeted_structure("") + with self.assertRaises(ValueError): + self.mapper._get_targeted_structure("mystery") + with self.assertRaises(ValueError): + self.mapper._get_targeted_structure(999) + with self.assertRaises(TypeError): + self.mapper._get_targeted_structure(3.14) + + def test_fov_reference_and_dimensions_helpers(self) -> None: + """Coordinate and dimension helpers should validate required inputs.""" + with self.assertRaises(ValueError): + self.mapper._get_fov_translation_mm({"fov_reference": "Lambda"}) + with self.assertRaises(ValueError): + self.mapper._get_image_dimensions([]) + + def test_single_plane_group_maps_plane_without_notes(self) -> None: + """Single-plane groups should emit Plane and allow empty stream notes.""" + metadata = copy.deepcopy(self.fixture) + metadata["job_settings"].pop("magnification") + plane = metadata["session_metadata"]["platform"]["imaging_plane_groups"][0]["imaging_planes"][0] + plane.pop("scanimage_roi_index") + metadata["session_metadata"]["platform"]["imaging_plane_groups"][0]["imaging_planes"] = [plane] + metadata["session_metadata"]["platform"]["imaging_plane_groups"][0]["scanimage_power_percent"] = 42 + metadata["session_metadata"]["platform"]["imaging_plane_groups"][0].pop("scanimage_split_percent") + + acquisition = self.mapper.transform(metadata) + image_plane = acquisition.data_streams[0].configurations[0].images[0].planes[0] + + self.assertEqual(type(image_plane).__name__, "Plane") + self.assertIsNone(acquisition.data_streams[0].notes) + + def test_empty_plane_groups_raise(self) -> None: + """Mapper should reject empty plane group lists.""" + metadata = copy.deepcopy(self.fixture) + metadata["session_metadata"]["platform"]["imaging_plane_groups"] = [] + + with self.assertRaises(ValueError) as context: + self.mapper.transform(metadata) + + self.assertIn("must contain at least one group", str(context.exception)) + + def test_missing_power_raises(self) -> None: + """Mapper should reject missing percent power.""" + metadata = copy.deepcopy(self.fixture) + group = metadata["session_metadata"]["platform"]["imaging_plane_groups"][0] + group.pop("scanimage_power_percent") + + with self.assertRaises(ValueError) as context: + self.mapper.transform(metadata) + + self.assertIn("scanimage_power_percent", str(context.exception)) + + def test_performance_metrics_helper(self) -> None: + """Performance metrics helper should infer reward units when needed.""" + metrics = self.mapper._build_performance_metrics( + { + "reward_consumed_during_epoch": 1.5, + "reward_consumed_unit": None, + "trials_total": 10, + "output_parameters": {"task_parameters": {"stage_in_use": "stage-a"}}, + } + ) + self.assertEqual(metrics.reward_consumed_unit, "milliliter") + self.assertEqual(metrics.trials_total, 10) + + def test_code_helper_variants(self) -> None: + """Code helper should handle multiple parameter shapes and errors.""" + multi_code = self.mapper._build_code( + { + "stimulus_parameters": [ + {"stimulus_name": "A", "value": 1}, + {"stimulus_name": "A", "value": 2}, + ], + "script": {"name": "script", "version": "1.0", "url": "", "parameters": {"extra": True}}, + "software": [{"name": "camstim", "version": "1.0"}], + } + ) + self.assertEqual(multi_code.parameters.model_dump()["A"]["value"], 1) + self.assertEqual(multi_code.parameters.model_dump()["A_1"]["value"], 2) + self.assertEqual(multi_code.parameters.model_dump()["A"]["stimulus_type"], "Visual Stimulation") + self.assertIsNone(multi_code.parameters.model_dump()["A"]["notes"]) + self.assertTrue(multi_code.parameters.model_dump()["extra"]) + + preserved_code = self.mapper._build_code( + { + "stimulus_parameters": [ + { + "stimulus_name": "B", + "stimulus_parameters": {"contrast": [1.0]}, + "stimulus_template_name": [], + "stimulus_type": "Custom visual", + "notes": "keep me", + } + ] + } + ) + self.assertEqual(preserved_code.parameters.model_dump()["stimulus_type"], "Custom visual") + self.assertEqual(preserved_code.parameters.model_dump()["notes"], "keep me") + + direct_code = self.mapper._build_code({"stimulus_parameters": {"value": 1}}) + self.assertEqual(direct_code.name, "Code") + self.assertEqual(direct_code.parameters.model_dump()["value"], 1) + + self.assertIsNone(self.mapper._build_code({})) + self.assertEqual( + self.mapper._build_code( + {"script": {"name": "script", "version": "1.0", "url": "", "parameters": {"extra": True}}} + ).parameters.model_dump()["extra"], + True, + ) + + with self.assertRaises(TypeError): + self.mapper._build_code({"stimulus_parameters": ["bad"]}) + with self.assertRaises(TypeError): + self.mapper._build_code({"stimulus_parameters": [{"value": 1}, "bad"]}) + with self.assertRaises(TypeError): + self.mapper._build_code( + { + "stimulus_parameters": 5, + "script": {"name": "script", "version": "1.0", "url": "", "parameters": {"extra": True}}, + } + ) + + +if __name__ == "__main__": + unittest.main() From ec458e25a77858b128bd540d3580269385220130 Mon Sep 17 00:00:00 2001 From: seanmcculloch Date: Thu, 8 Oct 2026 10:35:50 -0700 Subject: [PATCH 2/3] test(mesoscope): add package init so tests are discovered --- tests/mesoscope/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/mesoscope/__init__.py diff --git a/tests/mesoscope/__init__.py b/tests/mesoscope/__init__.py new file mode 100644 index 00000000..04332bf6 --- /dev/null +++ b/tests/mesoscope/__init__.py @@ -0,0 +1 @@ +"""Tests for mesoscope mapper module.""" From c3c407911935ed274b1deabe0c3e40f40d0c31c1 Mon Sep 17 00:00:00 2001 From: seanmcculloch Date: Fri, 16 Oct 2026 11:10:48 -0700 Subject: [PATCH 3/3] fix(mesoscope): read camstim_epochs from extractor 0.3.13 --- pyproject.toml | 2 +- src/aind_metadata_mapper/mesoscope/README.md | 2 +- src/aind_metadata_mapper/mesoscope/mapper.py | 4 ++-- tests/fixtures/mesoscope_intermediate.json | 2 +- tests/mesoscope/test_mapper.py | 20 ++++++++++---------- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 750b8337..e9c638fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "pydantic-settings>=2.8.0", "jsonschema", "aind-data-schema>=2.7.1,<3", - "aind-metadata-extractor==0.3.11", + "aind-metadata-extractor==0.3.13", ] [project.optional-dependencies] diff --git a/src/aind_metadata_mapper/mesoscope/README.md b/src/aind_metadata_mapper/mesoscope/README.md index 8a2b9cf6..1b1213b2 100644 --- a/src/aind_metadata_mapper/mesoscope/README.md +++ b/src/aind_metadata_mapper/mesoscope/README.md @@ -6,7 +6,7 @@ This mapper transforms extractor-produced `mesoscope.json` into ## Input contract The mapper consumes the `aind-metadata-extractor` `MesoscopeExtractModel`, -including `tiff_header`, `session_metadata`, `camstim_epchs`, +including `tiff_header`, `session_metadata`, `camstim_epochs`, `camstim_session_type`, and `job_settings`. `job_settings.instrument_id` is required. Preparation should populate this diff --git a/src/aind_metadata_mapper/mesoscope/mapper.py b/src/aind_metadata_mapper/mesoscope/mapper.py index 2a06ed86..61b791a7 100644 --- a/src/aind_metadata_mapper/mesoscope/mapper.py +++ b/src/aind_metadata_mapper/mesoscope/mapper.py @@ -485,7 +485,7 @@ def _build_stimulus_epoch(self, epoch: dict, fallback_tz: tzinfo) -> StimulusEpo return StimulusEpoch( stimulus_start_time=stimulus_start, stimulus_end_time=stimulus_end, - stimulus_name=self._get_required_value(epoch, "stimulus_name", "camstim_epchs"), + stimulus_name=self._get_required_value(epoch, "stimulus_name", "camstim_epochs"), stimulus_modalities=self._resolve_stimulus_modalities(epoch), performance_metrics=self._build_performance_metrics(epoch), code=self._build_code(epoch), @@ -557,7 +557,7 @@ def transform(self, metadata: dict) -> Acquisition: stimulus_epochs = [ self._build_stimulus_epoch(epoch, session_start.tzinfo or PACIFIC_TIMEZONE) - for epoch in raw_metadata.get("camstim_epchs", []) + for epoch in raw_metadata.get("camstim_epochs", []) ] acquisition_start, acquisition_end = self._get_acquisition_bounds(session_start, session_end, stimulus_epochs) diff --git a/tests/fixtures/mesoscope_intermediate.json b/tests/fixtures/mesoscope_intermediate.json index feac53d4..5c3422ee 100644 --- a/tests/fixtures/mesoscope_intermediate.json +++ b/tests/fixtures/mesoscope_intermediate.json @@ -64,7 +64,7 @@ } } }, - "camstim_epchs": [ + "camstim_epochs": [ { "stimulus_start_time": "2026-02-12T13:05:03.110704-08:00", "stimulus_end_time": "2026-02-12T13:19:02.807314-08:00", diff --git a/tests/mesoscope/test_mapper.py b/tests/mesoscope/test_mapper.py index 2cdd3a71..3a24ff5f 100644 --- a/tests/mesoscope/test_mapper.py +++ b/tests/mesoscope/test_mapper.py @@ -151,8 +151,8 @@ def test_transform_assigns_pacific_timezone_to_naive_times(self) -> None: metadata = copy.deepcopy(self.fixture) metadata["job_settings"]["session_start_time"] = "2026-02-12T13:04:36.555247" metadata["job_settings"]["session_end_time"] = "2026-02-12T14:07:12.672057" - metadata["camstim_epchs"][0]["stimulus_start_time"] = "2026-02-12T13:05:03.110704" - metadata["camstim_epchs"][0]["stimulus_end_time"] = "2026-02-12T13:19:02.807314" + metadata["camstim_epochs"][0]["stimulus_start_time"] = "2026-02-12T13:05:03.110704" + metadata["camstim_epochs"][0]["stimulus_end_time"] = "2026-02-12T13:19:02.807314" acquisition = self.mapper.transform(metadata) @@ -222,8 +222,8 @@ def test_transform_extends_acquisition_to_stimulus_interval(self) -> None: """Acquisition bounds should span stream and stimulus times.""" metadata = copy.deepcopy(self.fixture) metadata["job_settings"]["session_end_time"] = "2026-02-20T11:17:49.592634-08:00" - metadata["camstim_epchs"][0]["stimulus_start_time"] = "2026-02-20T10:07:44.569904-08:00" - metadata["camstim_epchs"][0]["stimulus_end_time"] = "2026-02-20T11:17:55.926134-08:00" + metadata["camstim_epochs"][0]["stimulus_start_time"] = "2026-02-20T10:07:44.569904-08:00" + metadata["camstim_epochs"][0]["stimulus_end_time"] = "2026-02-20T11:17:55.926134-08:00" acquisition = self.mapper.transform(metadata) @@ -240,8 +240,8 @@ def test_transform_extends_acquisition_to_stimulus_interval(self) -> None: def test_transform_rejects_inverted_stimulus_interval(self) -> None: """Mapper should reject inverted stimulus intervals.""" metadata = copy.deepcopy(self.fixture) - metadata["camstim_epchs"][0]["stimulus_start_time"] = "2026-02-12T13:19:02.807314-08:00" - metadata["camstim_epchs"][0]["stimulus_end_time"] = "2026-02-12T13:05:03.110704-08:00" + metadata["camstim_epochs"][0]["stimulus_start_time"] = "2026-02-12T13:19:02.807314-08:00" + metadata["camstim_epochs"][0]["stimulus_end_time"] = "2026-02-12T13:05:03.110704-08:00" with self.assertRaises(ValueError) as context: self.mapper.transform(metadata) @@ -251,7 +251,7 @@ def test_transform_rejects_inverted_stimulus_interval(self) -> None: def test_transform_defaults_missing_stimulus_modalities_to_visual(self) -> None: """Mapper should normalize missing mesoscope Camstim modalities to Visual.""" metadata = copy.deepcopy(self.fixture) - metadata["camstim_epchs"][0].pop("stimulus_modalities", None) + metadata["camstim_epochs"][0].pop("stimulus_modalities", None) acquisition = self.mapper.transform(metadata) @@ -260,9 +260,9 @@ def test_transform_defaults_missing_stimulus_modalities_to_visual(self) -> None: def test_transform_preserves_explicit_stimulus_modalities(self) -> None: """Mapper should preserve explicit upstream stimulus fields.""" metadata = copy.deepcopy(self.fixture) - metadata["camstim_epchs"][0]["stimulus_modalities"] = [StimulusModality.VISUAL] - metadata["camstim_epchs"][0]["stimulus_parameters"][0]["stimulus_type"] = "Custom visual" - metadata["camstim_epchs"][0]["stimulus_parameters"][0]["notes"] = "keep me" + metadata["camstim_epochs"][0]["stimulus_modalities"] = [StimulusModality.VISUAL] + metadata["camstim_epochs"][0]["stimulus_parameters"][0]["stimulus_type"] = "Custom visual" + metadata["camstim_epochs"][0]["stimulus_parameters"][0]["notes"] = "keep me" acquisition = self.mapper.transform(metadata)