Edit the unit test for notebookC

good_standard_metagv0.csv

@@ -0,0 +1,38 @@
+[Header],,,,,,,,
+IEMFileVersion,4,,,,,,,
+SheetType,tellseq_metag,,,,,,,
+SheetVersion,10,,,,,,,
+Investigator Name,Knight,,,,,,,
+Experiment Name,RKL_experiment,,,,,,,
+Date,2025-08-14,,,,,,,
+Workflow,GenerateFASTQ,,,,,,,
+Application,FASTQ Only,,,,,,,
+Assay,Metagenomic,,,,,,,
+Description,,,,,,,,
+Chemistry,Default,,,,,,,
+,,,,,,,,
+[Reads],,,,,,,,
+151,,,,,,,,
+151,,,,,,,,
+,,,,,,,,
+[Settings],,,,,,,,
+ReverseComplement,0,,,,,,,
+,,,,,,,,
+[Data],,,,,,,,
+Sample_ID,Sample_Name,Sample_Plate,well_id_384,barcode_id,Sample_Project,Well_description,Lane,
+sample_1,sample.1,A1,False,C501,MyProject_99999,A1.sample.1.False,1,
+sample_2,sample.2,A2,False,C509,MyProject_99999,A2.sample.2.False,1,
+sample_3,sample.3,A3,False,C520,MyProject_99999,A3.sample.3.False,1,
+,,,,,,,,
+[Bioinformatics],,,,,,,,
+Sample_Project,QiitaID,BarcodesAreRC,ForwardAdapter,ReverseAdapter,HumanFiltering,library_construction_protocol,experiment_design_description,contains_replicates
+MyProject_99999,99999,True,AACC,GGTT,False,some protocol,some description,False
+,,,,,,,,
+[Contact],,,,,,,,
+Sample_Project,Email,,,,,,,
+MyProject_99999,foo@bar.org,,,,,,,
+,,,,,,,,
+[SampleContext],,,,,,,,
+sample_name,sample_type,primary_qiita_study,secondary_qiita_studies,,,,,
+sample.3,control blank,99999,,,,,,
+,,,,,,,,

notebooks/tellseq_D_variable_volume_pooling.ipynb

@@ -375,6 +375,7 @@
    "outputs": [],
    "source": [
     "## INPUT\n",
+    "# when paramaters does not exist use the variable value, and if exhists, use the key in the test dic.\n",
     "iseqnormed_picklist_fbase = assign_input(variable_value='./test_output/Pooling/Tellseq_iSeqnormpool',\n",
     "                               test_dict=test_dict,\n",
     "                               key='iseqnormed_picklist_fbase')"

notebooks/test_output/SampleSheets/Tellseq_samplesheet_instrument_iseq_set_col19to24.csv

@@ -2,7 +2,7 @@
 Experiment Name,RKLtest_col19to24,,,,,,,,,,
 Investigator Name,Enter the investigator name (optional),,,,,,,,,,
 Project Name,TestProjA_10002_1_2_3_4_10to1dilution,,,,,,,,,,
-Date,2025-04-01,,,,,,,,,,
+Date,2025-09-04,,,,,,,,,,
 Workflow,GenerateFASTQ,,,,,,,,,,
 Library Prep Kit,TELLSEQ,,,,,,,,,,
 [Manifest],,,,,,,,,,,

notebooks/tests/test2_tellseq_C_equal_volume_pooling.py

@@ -0,0 +1,159 @@
+import unittest
+import papermill as pm
+import tempfile
+from pathlib import Path
+import os
+import shutil
+
+NOTEBOOK = "tellseq_C_equal_volume_pooling.ipynb"
+
+# Golden files live under notebooks/test_output/<subdir>/<filename>
+EXPECTED_SUBDIRS = ["Pooling", "Indices", "SampleSheets", "QC"]
+
+# Only compare data files (skip .ipynb, images, etc.)
+DATA_EXTS = {".txt", ".csv", ".tsv"}
+
+
+class TestTellseqC_ShadowWorkspace(unittest.TestCase):
+    def setUp(self):
+        # project root (parent of this test file’s folder)
+        self.notebooks_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        self.repo_test_output_dir = os.path.join(self.notebooks_dir, "test_output")
+
+        # sanity: required inputs exist in repo
+        must_inputs = [
+            os.path.join(self.repo_test_output_dir, "QC", "Tellseq_plate_df_B.txt"),
+            os.path.join(self.repo_test_output_dir, "QC", "Tellseq_expt_info.yml"),
+        ]
+        for fp in must_inputs:
+            if not os.path.isfile(fp):
+                raise FileNotFoundError(f"[SETUP] Required input not found in repo: {fp}")
+
+    def _find_expected_path(self, filename: str) -> str:
+        """Find golden file by filename under test_output/<subdir>/*."""
+        for sub in EXPECTED_SUBDIRS:
+            candidate = os.path.join(self.repo_test_output_dir, sub, filename)
+            if os.path.exists(candidate):
+                return candidate
+        raise AssertionError(
+            f"[COMPARE] Expected golden file not found for '{filename}'. "
+            f"Searched under: {', '.join(EXPECTED_SUBDIRS)}"
+        )
+
+    def _assert_text_files_equal(self, produced_fp: Path, expected_fp: Path):
+        """Strict line-by-line text comparison with helpful diffs."""
+        with open(produced_fp, "r") as f_out, open(expected_fp, "r") as f_exp:
+            out_lines = f_out.readlines()
+            exp_lines = f_exp.readlines()
+
+        self.assertEqual(
+            len(out_lines), len(exp_lines),
+            msg=(f"[DIFF] Line count differs for '{produced_fp.name}': "
+                 f"produced={len(out_lines)} vs expected={len(exp_lines)}")
+        )
+
+        for i, (o, e) in enumerate(zip(out_lines, exp_lines), start=1):
+            if o != e:
+                self.fail(
+                    f"[DIFF] Mismatch in '{produced_fp.name}' at line {i}.\n"
+                    f"Produced: {o.rstrip()}\n"
+                    f"Expected: {e.rstrip()}\n"
+                )
+
+    def test_equal_volume_pooling_outputs_shadow(self):
+        """
+        Keep Notebook C's hardcoded './test_output/...' as-is by creating that
+        structure under a temp dir and running with cwd=tmp. Then compare tmp outputs
+        vs repo goldens by filename.
+        """
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            tmp_root = Path(tmp_dir)            # shadow root = './'
+            tmp_out  = tmp_root / "test_output" # ./test_output
+            tmp_qc   = tmp_out / "QC"
+            tmp_idx  = tmp_out / "Indices"
+            tmp_ss   = tmp_out / "SampleSheets"
+
+            # create expected dirs under tmp
+            for d in (tmp_qc, tmp_idx, tmp_ss):
+                d.mkdir(parents=True, exist_ok=True)
+
+            # copy required inputs from repo -> tmp
+            shutil.copy2(
+                os.path.join(self.repo_test_output_dir, "QC", "Tellseq_plate_df_B.txt"),
+                tmp_qc / "Tellseq_plate_df_B.txt",
+            )
+            shutil.copy2(
+                os.path.join(self.repo_test_output_dir, "QC", "Tellseq_expt_info.yml"),
+                tmp_qc / "Tellseq_expt_info.yml",
+            )
+
+
+            run_params = {
+                "current_set_id": "col19to24",
+                "total_vol": 190,
+                "iseq_sequencer": "iSeq",
+                "novaseq_sequencer": "NovaSeqXPlus",
+                "full_plate_fp": "./test_output/QC/Tellseq_plate_df_B.txt",
+                "expt_config_fp": "./test_output/QC/Tellseq_expt_info.yml",
+            }
+
+            # run notebook with cwd switched to tmp_root so './test_output' points to tmp
+            prev_cwd = os.getcwd()
+            try:
+                os.chdir(tmp_root)
+                pm.execute_notebook(
+                    input_path=os.path.join(self.notebooks_dir, NOTEBOOK),
+                    output_path=str(tmp_root / "executed_test_equal_volume_pooling.ipynb"),
+                    parameters=run_params,
+                    log_output=True,
+                )
+            finally:
+                os.chdir(prev_cwd)
+
+            # collect produced data files under tmp/test_output (skip .ipynb)
+            produced_files = [
+                p for p in tmp_root.glob("test_output/**/*")
+                if p.is_file() and p.suffix.lower() in DATA_EXTS
+            ]
+            self.assertTrue(
+                produced_files,
+                msg="[CHECK] No data outputs (.txt/.csv/.tsv) were produced under tmp/test_output."
+            )
+
+            # --- Expect exactly the names you observed were produced ---
+            expected_names = {
+                "Tellseq_evp_set_col19to24.txt",
+                "Tellseq_plate_df_B.txt",
+                "Tellseq_plate_df_C_set_col19to24.txt",
+                "Tellseq_samplesheet_instrument_iseq_set_col19to24.csv",
+                "Tellseq_samplesheet_spp_iseq_set_col19to24.csv",
+                "Tellseq_samplesheet_spp_novaseqxplus_set_col19to24.csv",
+            }
+
+            # helpful once: see what was actually produced
+            print("Produced files:", sorted(p.name for p in produced_files))
+
+            produced_names = {p.name for p in produced_files}
+            missing = expected_names - produced_names
+            extra   = produced_names - expected_names
+            self.assertFalse(missing, f"Missing outputs: {sorted(missing)}")
+            self.assertFalse(extra,   f"Unexpected outputs: {sorted(extra)}")
+
+            # compare each produced file to its golden
+            for name in sorted(expected_names):
+                produced_fp = next(p for p in produced_files if p.name == name)
+                expected_fp = self._find_expected_path(name)
+
+                # guard: prevent self-compare if paths somehow coincide
+                if Path(expected_fp).resolve() == produced_fp.resolve():
+                    self.fail(
+                        "[GUARD] Produced file path equals expected golden path:\n"
+                        f"  {produced_fp}\n"
+                        "Ensure notebook wrote into tmp and goldens remain in repo."
+                    )
+
+                self._assert_text_files_equal(produced_fp, Path(expected_fp))
+
+
+if __name__ == "__main__":
+    unittest.main()

notebooks/tests/test_tellseq_C_equal_volume_pooling_1.py

@@ -0,0 +1,70 @@
+import unittest
+import papermill as pm
+import tempfile
+from pathlib import Path
+import os
+
+NOTEBOOK = "tellseq_C_equal_volume_pooling.ipynb"
+PICKLIST_FNAME = "Tellseq_iSeqnormpool_set_col19to24.txt"
+
+
+class TestTellseqD(unittest.TestCase):
+    def setUp(self):
+        self.notebooks_dir = os.path.dirname(os.path.dirname(__file__))
+        self.test_output_dir = os.path.join(self.notebooks_dir, 'test_output')
+
+    def test_iseqnorm_picklist(self):
+        """Verify notebook produces expected output for iSeqnormed picklist."""
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            tmp_path = Path(tmp_dir)
+
+            run_params = {
+                    """
+                    'plate_df_set_fp': f"{self.test_output_dir}/QC/"
+                                       f"Tellseq_plate_df_C_set_col19to24.txt",
+                    'read_counts_fps': [
+                        f"{self.notebooks_dir}/test_data/Demux/"
+                        f"Tellseq_fastqc_sequence_counts.tsv"],
+                    'dynamic_range': 5,
+                    'iseqnormed_picklist_fbase':
+                        f"{tmp_path}/Tellseq_iSeqnormpool"
+                    """
+                    'full_plate_fp' : f"{self.test_output_dir}/QC/Tellseq_plate_df_B.txt",
+                    'expt_config_fp' : f"{self.test_output_dir}/QC/Tellseq_expt_info.yml",
+                    'current_set_id' : "col19to24",
+                    'total_vol' : 190,
+                    'evp_picklist_fbase' : f"{self.test_output_dir}/Indices/Tellseq_evp",
+                    'machine_samplesheet_fbase' : f"{self.test_output_dir}/SampleSheets/Tellseq_samplesheet_instrument_iseq",
+                    'spp_samplesheet_fbase' : f"{self.test_output_dir}/SampleSheets/Tellseq_samplesheet_spp",
+                    'iseq_sequencer' : 'iSeq',
+                    'novaseq_sequencer' : 'NovaSeqXPlus'
+
+            }
+
+            pm.execute_notebook(
+                input_path=f"{self.notebooks_dir}/{NOTEBOOK}",
+                output_path=f"{tmp_path}/test_iseqnorm_picklist.ipynb",
+                parameters=run_params,
+                log_output=True,
+            )
+
+            out_iseqnormed_picklist_fp = f"{tmp_path}/{PICKLIST_FNAME}"
+            self.assertTrue(os.path.exists(out_iseqnormed_picklist_fp),
+                            msg="Notebook did not produce desired file.")
+
+            exp_iseqnormed_fp = \
+                f"{self.test_output_dir}/Pooling/{PICKLIST_FNAME}"
+            with open(out_iseqnormed_picklist_fp, 'r') as out:
+                with open(exp_iseqnormed_fp, 'r') as test:
+                    out_lines = out.readlines()
+                    test_lines = test.readlines()
+                    for out_line, test_line in zip(out_lines,
+                                                   test_lines):
+                        self.assertEqual(out_line, test_line,
+                                         msg=("Lines of output" +
+                                              "and test don't match"))
+
+
+if __name__ == "__main__":
+    unittest.main()

notebooks/tests/test_tellseq_D_variable_volume_pooling.py

@@ -1,26 +1,36 @@
+'''
+1.Checking whether the expected output file (Tellseq_iSeqnormpool_set_col19to24.txt) was created.
+2.Comparing the contents of that file to a known "expected" version of the file.
+'''
+
+# imports are executed
 import unittest
 import papermill as pm
 import tempfile
 from pathlib import Path
 import os
 
+# define constants variable
 NOTEBOOK = "tellseq_D_variable_volume_pooling.ipynb"
 PICKLIST_FNAME = "Tellseq_iSeqnormpool_set_col19to24.txt"
 
-
+# class definition is executed
 class TestTellseqD(unittest.TestCase):
+    #method 1. setup
+    # automatically before every test_... method-
     def setUp(self):
-        self.notebooks_dir = os.path.dirname(os.path.dirname(__file__))
-        self.test_output_dir = os.path.join(self.notebooks_dir, 'test_output')
+        self.notebooks_dir = os.path.dirname(os.path.dirname(__file__)) # /Users/seosonghee/Documents/KnightLab/kl-metapool/notebooks
+        self.test_output_dir = os.path.join(self.notebooks_dir, 'test_output') # the  "<project_root>/test_output"
 
-    def test_iseqnorm_picklist(self):
-        """Verify notebook produces expected output for iSeqnormed picklist."""
+    #method 2. test_iseqnorm_picklist (Starts with test_)
+        # creates the expected output file, 
+        # matches an expected “golden” file line-by-line.
 
+    def test_iseqnorm_picklist(self):
         with tempfile.TemporaryDirectory() as tmp_dir:
             tmp_path = Path(tmp_dir)
-
+            # Build the parameters to inject into the notebook
             run_params = {
-                'test_dict': {
                     'plate_df_set_fp': f"{self.test_output_dir}/QC/"
                                        f"Tellseq_plate_df_C_set_col19to24.txt",
                     'read_counts_fps': [
@@ -29,16 +39,15 @@ def test_iseqnorm_picklist(self):
                     'dynamic_range': 5,
                     'iseqnormed_picklist_fbase':
                         f"{tmp_path}/Tellseq_iSeqnormpool"
-                }
             }
-
+            # run the notebook /test
             pm.execute_notebook(
                 input_path=f"{self.notebooks_dir}/{NOTEBOOK}",
                 output_path=f"{tmp_path}/test_iseqnorm_picklist.ipynb",
                 parameters=run_params,
                 log_output=True,
             )
-
+     # If the notebook didn’t write Tellseq_iSeqnormpool_set_col19to24.txt, the test fails here.
             out_iseqnormed_picklist_fp = f"{tmp_path}/{PICKLIST_FNAME}"
             self.assertTrue(os.path.exists(out_iseqnormed_picklist_fp),
                             msg="Notebook did not produce desired file.")
@@ -55,6 +64,6 @@ def test_iseqnorm_picklist(self):
                                          msg=("Lines of output" +
                                               "and test don't match"))
 
-
+# run the file directly
 if __name__ == "__main__":
     unittest.main()