From 4fd7d1239baa2479ec6d9a78e452376f956d3c52 Mon Sep 17 00:00:00 2001 From: Cameron Mattson Date: Wed, 12 Aug 2026 10:36:11 -0600 Subject: [PATCH 1/7] feat: add analysis csv source filtering --- .../cpg0016/analysis_csv_downloader.py | 81 +++++++++++++++++-- 1 file changed, 73 insertions(+), 8 deletions(-) diff --git a/src/jump_image_datasets/cpg0016/analysis_csv_downloader.py b/src/jump_image_datasets/cpg0016/analysis_csv_downloader.py index 4fc6ed4..8dadf87 100644 --- a/src/jump_image_datasets/cpg0016/analysis_csv_downloader.py +++ b/src/jump_image_datasets/cpg0016/analysis_csv_downloader.py @@ -10,6 +10,7 @@ from __future__ import annotations import os +import re import shutil import subprocess import tempfile @@ -49,6 +50,7 @@ "cytoplasm": "Cytoplasm.csv", "cytoplasm.csv": "Cytoplasm.csv", } +SOURCE_NAME_PATTERN = re.compile(r"^source_[^/]+$") @dataclass(frozen=True) @@ -258,6 +260,32 @@ def normalize_analysis_csv_filenames(csv_names: Sequence[str] | None) -> tuple[s return tuple(normalized_names) +def normalize_analysis_sources(sources: Sequence[str] | None) -> tuple[str, ...] | None: + """Normalize requested source names to canonical dataset source segments.""" + + if sources is None: + return None + + normalized_sources: list[str] = [] + invalid_sources: list[str] = [] + for source in sources: + normalized_source = str(source).strip() + if not SOURCE_NAME_PATTERN.fullmatch(normalized_source): + invalid_sources.append(str(source)) + continue + if normalized_source not in normalized_sources: + normalized_sources.append(normalized_source) + + if invalid_sources: + invalid_display = ", ".join(repr(source) for source in invalid_sources) + raise ValueError( + "Invalid analysis sources: " + f"{invalid_display}. Sources must match the dataset path segment format, for example 'source_10'." + ) + + return tuple(normalized_sources) + + def build_analysis_csv_sets_from_s3_urls( s3_urls: list[str], *, @@ -331,10 +359,18 @@ def build_analysis_csv_sets_from_local_paths( return csv_sets -def _build_analysis_csv_include_patterns(csv_filenames: Sequence[str]) -> list[str]: +def _build_analysis_csv_include_patterns( + csv_filenames: Sequence[str], + sources: Sequence[str] | None = None, +) -> list[str]: """Build AWS CLI include patterns for the requested analysis CSV filenames.""" - return [f"source_*/workspace/analysis/**/{filename}" for filename in csv_filenames] + source_patterns = tuple(sources) if sources is not None else ("source_*",) + return [ + f"{source_pattern}/workspace/analysis/**/{filename}" + for source_pattern in source_patterns + for filename in csv_filenames + ] def _create_aws_cli_config(max_concurrent_requests: int) -> str: @@ -467,7 +503,11 @@ def iter_analysis_csv_sets(self) -> Iterator[AnalysisCSVSet]: for csv_set in self.analysis_csv_sets: yield _copy_analysis_csv_set(csv_set) - def download_all_csv_profiles(self, csv_names: Sequence[str] | None = None) -> DownloadSummary: + def download_all_csv_profiles( + self, + csv_names: Sequence[str] | None = None, + sources: Sequence[str] | None = None, + ) -> DownloadSummary: """Download analysis CSV files into ``output_dir`` with the AWS CLI. This method uses the AWS CLI transfer manager against the public @@ -482,6 +522,10 @@ def download_all_csv_profiles(self, csv_names: Sequence[str] | None = None) -> D names such as ``image`` or ``nuclei`` as well as full filenames such as ``Image.csv`` and ``Nuclei.csv``. When omitted, all standard profile CSVs are downloaded. + sources + Optional subset of dataset source path segments to download, for + example ``["source_10", "source_11"]``. When omitted, all + non-``source_all`` sources are eligible. Returns ------- @@ -494,10 +538,18 @@ def download_all_csv_profiles(self, csv_names: Sequence[str] | None = None) -> D return DownloadSummary(total_jobs=0, downloaded=0, skipped=0, failed=0, failures=[]) requested_filenames = tuple(normalize_analysis_csv_filenames(csv_names)) + requested_sources = normalize_analysis_sources(sources) requested_filename_set = set(requested_filenames) - preexisting_paths = self._discover_local_analysis_csv_paths(requested_filename_set) - self._run_aws_analysis_csv_download(requested_filenames) - post_download_paths = self._discover_local_analysis_csv_paths(requested_filename_set) + requested_source_set = set(requested_sources) if requested_sources is not None else None + preexisting_paths = self._discover_local_analysis_csv_paths( + requested_filenames=requested_filename_set, + requested_sources=requested_source_set, + ) + self._run_aws_analysis_csv_download(requested_filenames, requested_sources) + post_download_paths = self._discover_local_analysis_csv_paths( + requested_filenames=requested_filename_set, + requested_sources=requested_source_set, + ) self.analysis_csv_urls = [self._local_analysis_csv_path_to_s3_url(path) for path in post_download_paths] self.analysis_csv_sets = build_analysis_csv_sets_from_s3_urls( @@ -519,6 +571,7 @@ def download_all_csv_profiles(self, csv_names: Sequence[str] | None = None) -> D def _discover_local_analysis_csv_paths( self, requested_filenames: Optional[set[str]] = None, + requested_sources: Optional[set[str]] = None, ) -> list[Path]: """Discover local analysis CSV files under ``output_dir``.""" @@ -531,6 +584,11 @@ def _discover_local_analysis_csv_paths( for local_path in sorted(analysis_root.rglob("workspace/analysis/**/*.csv")) if not is_source_all_path(local_path.relative_to(self.output_dir).as_posix()) and (requested_filenames is None or local_path.name in requested_filenames) + and ( + requested_sources is None + or extract_metadata_source_from_dataset_path(local_path.relative_to(self.output_dir)) + in requested_sources + ) ] return local_paths @@ -540,7 +598,11 @@ def _local_analysis_csv_path_to_s3_url(self, local_path: Path) -> str: relative_path = local_path.relative_to(self.output_dir).as_posix().lstrip("/") return f"s3://{CPG0016_BUCKET}/{relative_path}" - def _run_aws_analysis_csv_download(self, requested_filenames: Sequence[str]) -> None: + def _run_aws_analysis_csv_download( + self, + requested_filenames: Sequence[str], + requested_sources: Sequence[str] | None, + ) -> None: """Download the requested analysis CSVs with the AWS CLI transfer manager.""" aws_path = shutil.which("aws") @@ -564,7 +626,10 @@ def _run_aws_analysis_csv_download(self, requested_filenames: Sequence[str]) -> "--exclude", "source_all/*", ] - for include_pattern in _build_analysis_csv_include_patterns(requested_filenames): + for include_pattern in _build_analysis_csv_include_patterns( + requested_filenames, + requested_sources, + ): command.extend(["--include", include_pattern]) command.extend(["--no-sign-request", "--only-show-errors"]) From de18c5818117336d1c9e9c9fa2a83b6f608f8bfa Mon Sep 17 00:00:00 2001 From: Cameron Mattson Date: Wed, 12 Aug 2026 10:36:11 -0600 Subject: [PATCH 2/7] test: cover analysis csv source filtering --- tests/test_cpg0016_analysis_csv_downloader.py | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/tests/test_cpg0016_analysis_csv_downloader.py b/tests/test_cpg0016_analysis_csv_downloader.py index 4f3401c..1071da3 100644 --- a/tests/test_cpg0016_analysis_csv_downloader.py +++ b/tests/test_cpg0016_analysis_csv_downloader.py @@ -297,6 +297,74 @@ def side_effect(command, env, capture_output) -> None: ).exists() +def test_download_all_csv_profiles_can_filter_requested_sources(tmp_path, monkeypatch) -> None: + output_dir = tmp_path / "downloads" + + def side_effect(command, env, capture_output) -> None: + assert command.count("--include") == 1 + assert "source_10/workspace/analysis/**/Nuclei.csv" in command + assert "source_*/workspace/analysis/**/Nuclei.csv" not in command + _write_downloaded_csv( + output_dir, + "cpg0016-jump/source_10/workspace/analysis/run_a/plate_a/analysis/plate_a-A01-1/Nuclei.csv", + ) + _write_downloaded_csv( + output_dir, + "cpg0016-jump/source_11/workspace/analysis/run_b/plate_b/analysis/plate_b-B03-2/Nuclei.csv", + ) + + _install_fake_aws(monkeypatch, side_effect=side_effect) + + downloader = analysis_csv_downloader.CPG0016AnalysisCSVDownloader( + output_dir=output_dir, + parallel=False, + workers=1, + verbose=False, + ) + + summary = downloader.download_all_csv_profiles(csv_names=["nuclei"], sources=["source_10"]) + + assert summary.total_jobs == 1 + assert summary.downloaded == 1 + assert all("source_10" in str(csv_set.folder_relative_path) for csv_set in downloader.get_analysis_csv_sets()) + + +def test_download_all_csv_profiles_can_filter_multiple_sources(tmp_path, monkeypatch) -> None: + output_dir = tmp_path / "downloads" + + def side_effect(command, env, capture_output) -> None: + assert command.count("--include") == 2 + assert "source_10/workspace/analysis/**/Image.csv" in command + assert "source_11/workspace/analysis/**/Image.csv" in command + assert "source_*/workspace/analysis/**/Image.csv" not in command + _write_downloaded_csv( + output_dir, + "cpg0016-jump/source_10/workspace/analysis/run_a/plate_a/analysis/plate_a-A01-1/Image.csv", + ) + _write_downloaded_csv( + output_dir, + "cpg0016-jump/source_11/workspace/analysis/run_b/plate_b/analysis/plate_b-B03-2/Image.csv", + ) + + _install_fake_aws(monkeypatch, side_effect=side_effect) + + downloader = analysis_csv_downloader.CPG0016AnalysisCSVDownloader( + output_dir=output_dir, + parallel=False, + workers=1, + verbose=False, + ) + + summary = downloader.download_all_csv_profiles(csv_names=["image"], sources=["source_10", "source_11"]) + + assert summary.total_jobs == 2 + assert summary.downloaded == 2 + assert {csv_set.read_csv("Image.csv").loc[0, "Metadata_Source"] for csv_set in downloader.iter_analysis_csv_sets()} == { + "source_10", + "source_11", + } + + def test_download_all_csv_profiles_accepts_mixed_csv_name_forms(tmp_path, monkeypatch) -> None: output_dir = tmp_path / "downloads" @@ -328,6 +396,20 @@ def side_effect(command, env, capture_output) -> None: assert summary.downloaded == 2 +def test_download_all_csv_profiles_rejects_invalid_source_names(tmp_path, monkeypatch) -> None: + _install_fake_aws(monkeypatch) + + downloader = analysis_csv_downloader.CPG0016AnalysisCSVDownloader( + output_dir=tmp_path / "downloads", + parallel=False, + workers=1, + verbose=False, + ) + + with pytest.raises(ValueError, match="Invalid analysis sources: '10', 'source/10'"): + downloader.download_all_csv_profiles(csv_names=["image"], sources=["10", "source/10"]) + + def test_download_all_csv_profiles_rejects_invalid_csv_names(tmp_path, monkeypatch) -> None: _install_fake_aws(monkeypatch) @@ -342,6 +424,27 @@ def test_download_all_csv_profiles_rejects_invalid_csv_names(tmp_path, monkeypat downloader.download_all_csv_profiles(csv_names=["not_a_csv"]) +def test_discover_local_analysis_csv_paths_can_filter_requested_sources(tmp_path) -> None: + _copy_test_csv_tree(tmp_path / "downloads") + + downloader = analysis_csv_downloader.CPG0016AnalysisCSVDownloader( + output_dir=tmp_path / "downloads", + parallel=False, + workers=1, + verbose=False, + use_existing_csvs_without_s3_check=True, + ) + + local_paths = downloader._discover_local_analysis_csv_paths( + requested_filenames={"Image.csv", "Nuclei.csv", "Cells.csv"}, + requested_sources={"source_10"}, + ) + + assert local_paths + assert all("/source_10/" in path.as_posix() for path in local_paths) + assert all("source_all" not in path.as_posix() for path in local_paths) + + def test_analysis_csv_set_read_csv_overwrites_metadata_source_from_s3_path(tmp_path, monkeypatch) -> None: output_dir = tmp_path / "downloads" From 9e2d48fb973f5a05d752a9e9fa08c88bc376e56d Mon Sep 17 00:00:00 2001 From: Cameron Mattson Date: Wed, 12 Aug 2026 10:36:11 -0600 Subject: [PATCH 3/7] docs: show analysis csv source filtering --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 25f2ee3..6e1a0f9 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,13 @@ nuclei_and_image_summary = downloader.download_all_csv_profiles( ) print(nuclei_and_image_summary) +# Or limit the download to specific dataset sources. +source_10_nuclei_summary = downloader.download_all_csv_profiles( + csv_names=["nuclei"], + sources=["source_10"], +) +print(source_10_nuclei_summary) + # Later, reuse the existing local CSV tree without checking S3. local_only_downloader = CPG0016AnalysisCSVDownloader( output_dir="downloaded_cpg0016_profiles", From a2ce02fbbb68d23b94cf74f8ef94a5ec6ea6a7cc Mon Sep 17 00:00:00 2001 From: Cameron Mattson Date: Wed, 12 Aug 2026 10:37:46 -0600 Subject: [PATCH 4/7] docs: add source-filtered profile download examples --- .../download_cpg0016_profiles_examples.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/nbconverted/download_cpg0016_profiles_examples.py b/docs/nbconverted/download_cpg0016_profiles_examples.py index 79e7320..29eb885 100644 --- a/docs/nbconverted/download_cpg0016_profiles_examples.py +++ b/docs/nbconverted/download_cpg0016_profiles_examples.py @@ -30,6 +30,24 @@ print(subset_summary) +# Limit downloads to specific dataset sources. Sources must use full names such +# as ``source_10``. +source_subset_summary = downloader.download_all_csv_profiles( + csv_names=["nuclei"], + sources=["source_10"], +) +print(source_subset_summary) + + +# Combine CSV-name and source filters when you only need a few profile tables +# from a few sources. +multi_source_summary = downloader.download_all_csv_profiles( + csv_names=["image", "nuclei"], + sources=["source_10", "source_11"], +) +print(multi_source_summary) + + # Reuse the existing local CSV tree later without checking S3 or probing which # files already exist remotely. local_only_downloader = CPG0016AnalysisCSVDownloader( From 89094992c6ab342d34be75704651f910301e0b94 Mon Sep 17 00:00:00 2001 From: Cameron Mattson Date: Wed, 12 Aug 2026 14:23:27 -0600 Subject: [PATCH 5/7] docs: expand analysis csv helper docstrings --- .../cpg0016/analysis_csv_downloader.py | 74 ++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/src/jump_image_datasets/cpg0016/analysis_csv_downloader.py b/src/jump_image_datasets/cpg0016/analysis_csv_downloader.py index 8dadf87..193c26b 100644 --- a/src/jump_image_datasets/cpg0016/analysis_csv_downloader.py +++ b/src/jump_image_datasets/cpg0016/analysis_csv_downloader.py @@ -90,7 +90,19 @@ class AnalysisCSVSet: other_local_paths: dict[str, Path] = field(default_factory=dict) def read_csv(self, filename: str) -> pd.DataFrame: - """Read one analysis CSV and overwrite ``Metadata_Source`` from its path.""" + """Read one analysis CSV and set ``Metadata_Source`` from its dataset path. + + Parameters + ---------- + filename + Analysis CSV filename to load from this grouped record. + + Returns + ------- + pandas.DataFrame + Loaded CSV contents with ``Metadata_Source`` overwritten from the + dataset-relative source segment encoded in the S3 or local path. + """ local_path = self._get_local_path(filename) source_path = self._get_source_path(filename) @@ -99,18 +111,64 @@ def read_csv(self, filename: str) -> pd.DataFrame: return dataframe def _get_local_path(self, filename: str) -> Path: + """Return the local path for one CSV filename or raise if it is unavailable. + + Parameters + ---------- + filename + Analysis CSV filename to resolve within this grouped record. + + Returns + ------- + pathlib.Path + Local filesystem path for the requested CSV. + + Raises + ------ + ValueError + If the requested CSV is not present in this grouped record. + """ + local_path = self._get_local_path_or_none(filename) if local_path is None: raise ValueError(f"CSV not available in this analysis set: {filename}") return local_path def _get_source_path(self, filename: str) -> str: + """Return the best provenance path for one CSV, preferring the S3 URL. + + Parameters + ---------- + filename + Analysis CSV filename to resolve within this grouped record. + + Returns + ------- + str + S3 URL when available, otherwise the local filesystem path for the + requested CSV. + """ + s3_url = self._get_s3_url_or_none(filename) if s3_url is not None: return s3_url return str(self._get_local_path(filename)) def _get_local_path_or_none(self, filename: str) -> Optional[Path]: + """Return the local path for one CSV filename when present in this set. + + Parameters + ---------- + filename + Analysis CSV filename to resolve within this grouped record. + + Returns + ------- + pathlib.Path | None + Local filesystem path for the requested CSV, or ``None`` when this + grouped record does not include that file. + """ + if filename == "Image.csv": return self.image_local_path if filename == "Nuclei.csv": @@ -122,6 +180,20 @@ def _get_local_path_or_none(self, filename: str) -> Optional[Path]: return self.other_local_paths.get(filename) def _get_s3_url_or_none(self, filename: str) -> Optional[str]: + """Return the S3 URL for one CSV filename when present in this set. + + Parameters + ---------- + filename + Analysis CSV filename to resolve within this grouped record. + + Returns + ------- + str | None + S3 URL for the requested CSV, or ``None`` when this grouped record + does not include a remote URL for that file. + """ + if filename == "Image.csv": return self.image_s3_url if filename == "Nuclei.csv": From 7cde5d376cb1ca3ec7bc9601552969606873ca16 Mon Sep 17 00:00:00 2001 From: Cameron Mattson Date: Wed, 12 Aug 2026 14:38:04 -0600 Subject: [PATCH 6/7] feat: extract analysis csv batch metadata from paths --- .../cpg0016/analysis_csv_downloader.py | 52 +++++++++++++++---- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/src/jump_image_datasets/cpg0016/analysis_csv_downloader.py b/src/jump_image_datasets/cpg0016/analysis_csv_downloader.py index 193c26b..67801f5 100644 --- a/src/jump_image_datasets/cpg0016/analysis_csv_downloader.py +++ b/src/jump_image_datasets/cpg0016/analysis_csv_downloader.py @@ -90,7 +90,7 @@ class AnalysisCSVSet: other_local_paths: dict[str, Path] = field(default_factory=dict) def read_csv(self, filename: str) -> pd.DataFrame: - """Read one analysis CSV and set ``Metadata_Source`` from its dataset path. + """Read one analysis CSV and set path-derived metadata when available. Parameters ---------- @@ -100,14 +100,21 @@ def read_csv(self, filename: str) -> pd.DataFrame: Returns ------- pandas.DataFrame - Loaded CSV contents with ``Metadata_Source`` overwritten from the - dataset-relative source segment encoded in the S3 or local path. + Loaded CSV contents with ``Metadata_Source`` and ``Metadata_Batch`` + overwritten from the dataset path when those segments can be + extracted from the S3 or local provenance path. """ local_path = self._get_local_path(filename) source_path = self._get_source_path(filename) dataframe = pd.read_csv(local_path) - dataframe["Metadata_Source"] = extract_metadata_source_from_dataset_path(source_path) + try: + metadata_values = extract_metadata_from_dataset_path(source_path) + except ValueError: + return dataframe + + dataframe["Metadata_Source"] = metadata_values["Metadata_Source"] + dataframe["Metadata_Batch"] = metadata_values["Metadata_Batch"] return dataframe def _get_local_path(self, filename: str) -> Path: @@ -205,24 +212,49 @@ def _get_s3_url_or_none(self, filename: str) -> Optional[str]: return self.other_s3_urls.get(filename) -def extract_metadata_source_from_dataset_path(path_str: str | Path) -> str: - """Extract the CPG0016 source segment from an S3 URL or mirrored local path.""" +def _split_dataset_path_parts(path_str: str | Path) -> list[str]: + """Return normalized path segments for an S3 URL or mirrored local path.""" raw_path = str(path_str).strip() parsed = urlparse(raw_path) if parsed.scheme == "s3": if parsed.netloc != CPG0016_BUCKET: raise ValueError(f"Invalid CPG0016 dataset path: {path_str}") - path_parts = [part for part in parsed.path.split("/") if part] - else: - path_parts = [part for part in raw_path.replace("\\", "/").split("/") if part] + return [part for part in parsed.path.split("/") if part] + return [part for part in raw_path.replace("\\", "/").split("/") if part] + + +def extract_metadata_from_dataset_path(path_str: str | Path) -> dict[str, str]: + """Extract CPG0016 source and batch segments from a dataset path.""" + + path_parts = _split_dataset_path_parts(path_str) try: prefix_index = path_parts.index(CPG0016_PREFIX) - return path_parts[prefix_index + 1] + source = path_parts[prefix_index + 1] + if path_parts[prefix_index + 2 : prefix_index + 4] != ["workspace", "analysis"]: + raise ValueError + batch = path_parts[prefix_index + 4] except (ValueError, IndexError) as exc: raise ValueError(f"Invalid CPG0016 dataset path: {path_str}") from exc + return { + "Metadata_Source": source, + "Metadata_Batch": batch, + } + + +def extract_metadata_source_from_dataset_path(path_str: str | Path) -> str: + """Extract the CPG0016 source segment from an S3 URL or mirrored local path.""" + + return extract_metadata_from_dataset_path(path_str)["Metadata_Source"] + + +def extract_metadata_batch_from_dataset_path(path_str: str | Path) -> str: + """Extract the CPG0016 batch segment from an S3 URL or mirrored local path.""" + + return extract_metadata_from_dataset_path(path_str)["Metadata_Batch"] + def _copy_analysis_csv_set(csv_set: AnalysisCSVSet) -> AnalysisCSVSet: """Return a defensive copy of one grouped CSV-set record.""" From b946c1d08a43d3931a3464d664b18980af02cfdf Mon Sep 17 00:00:00 2001 From: Cameron Mattson Date: Wed, 12 Aug 2026 14:38:07 -0600 Subject: [PATCH 7/7] test: cover analysis csv batch path metadata --- tests/test_cpg0016_analysis_csv_downloader.py | 45 +++++++++++++++++-- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/tests/test_cpg0016_analysis_csv_downloader.py b/tests/test_cpg0016_analysis_csv_downloader.py index 1071da3..b80293f 100644 --- a/tests/test_cpg0016_analysis_csv_downloader.py +++ b/tests/test_cpg0016_analysis_csv_downloader.py @@ -445,14 +445,16 @@ def test_discover_local_analysis_csv_paths_can_filter_requested_sources(tmp_path assert all("source_all" not in path.as_posix() for path in local_paths) -def test_analysis_csv_set_read_csv_overwrites_metadata_source_from_s3_path(tmp_path, monkeypatch) -> None: +def test_analysis_csv_set_read_csv_overwrites_metadata_source_and_batch_from_s3_path( + tmp_path, monkeypatch +) -> None: output_dir = tmp_path / "downloads" def side_effect(command, env, capture_output) -> None: _write_downloaded_csv( output_dir, "cpg0016-jump/source_10/workspace/analysis/run_a/plate_a/analysis/plate_a-A01-1/Image.csv", - "ImageNumber,Metadata_Source\n1,wrong_source\n", + "ImageNumber,Metadata_Source,Metadata_Batch\n1,wrong_source,wrong_batch\n", ) _install_fake_aws(monkeypatch, side_effect=side_effect) @@ -469,6 +471,7 @@ def side_effect(command, env, capture_output) -> None: dataframe = csv_set.read_csv("Image.csv") assert dataframe.loc[0, "Metadata_Source"] == "source_10" + assert dataframe.loc[0, "Metadata_Batch"] == "run_a" def test_download_all_csv_profiles_reports_existing_files_as_skipped(tmp_path, monkeypatch) -> None: @@ -587,13 +590,15 @@ def test_local_only_mode_uses_existing_downloads_without_s3(tmp_path, monkeypatc assert summary.skipped == 0 -def test_analysis_csv_set_read_csv_overwrites_metadata_source_from_local_path(tmp_path, monkeypatch) -> None: +def test_analysis_csv_set_read_csv_overwrites_metadata_source_and_batch_from_local_path( + tmp_path, monkeypatch +) -> None: local_image = ( tmp_path / "downloads/cpg0016-jump/source_11/workspace/analysis/run_b/plate_b/analysis/plate_b-B03-2/Image.csv" ) local_image.parent.mkdir(parents=True, exist_ok=True) - local_image.write_text("ImageNumber,Metadata_Source\n1,wrong_source\n") + local_image.write_text("ImageNumber,Metadata_Source,Metadata_Batch\n1,wrong_source,wrong_batch\n") monkeypatch.setattr( analysis_csv_downloader.s3fs, @@ -613,6 +618,38 @@ def test_analysis_csv_set_read_csv_overwrites_metadata_source_from_local_path(tm dataframe = csv_set.read_csv("Image.csv") assert dataframe.loc[0, "Metadata_Source"] == "source_11" + assert dataframe.loc[0, "Metadata_Batch"] == "run_b" + + +def test_analysis_csv_set_read_csv_skips_metadata_columns_for_malformed_s3_path(monkeypatch) -> None: + local_image = Path("/tmp/nonexistent-image.csv") + csv_set = analysis_csv_downloader.AnalysisCSVSet( + folder_relative_path=Path("cpg0016-jump/source_10/workspace/analysis/run_a/plate_a/analysis/plate_a-A01-1"), + folder_local_path=local_image.parent, + image_local_path=local_image, + image_s3_url="s3://cellpainting-gallery/cpg0016-jump/source_10/bad/run_a/Image.csv", + ) + + dataframe = analysis_csv_downloader.pd.DataFrame({"ImageNumber": [1]}) + monkeypatch.setattr( + analysis_csv_downloader.pd, + "read_csv", + lambda path: dataframe.copy(deep=True), + ) + + result = csv_set.read_csv("Image.csv") + + assert "Metadata_Source" not in result.columns + assert "Metadata_Batch" not in result.columns + + +def test_extract_metadata_batch_from_dataset_path_reads_analysis_batch_segment() -> None: + assert ( + analysis_csv_downloader.extract_metadata_batch_from_dataset_path( + "s3://cellpainting-gallery/cpg0016-jump/source_10/workspace/analysis/20211103-Run16/GR00004416/analysis/GR00004416-A01-3/Image.csv" + ) + == "20211103-Run16" + ) def test_constructor_requires_output_dir() -> None: