Feat/batch skip existing

docs/gap_roadmap.md

@@ -87,7 +87,7 @@ less often. The technical detail follows for whoever picks the item up.
 | 9 | Represent a localized casing hole | Experimental; static opening/isolation and one-year pressure-driven transfer verified |
 | 10 | Generate hole-transfer reporting from the workbook | Done; verified on CIRRUS (Smeaheia, one year) |
 | 11 | Align FIPLEG aquifer reporting with the current coarse-grid boundaries | Implemented; CIRRUS INIT check pending |
-| 12 | Run only new or unfinished workbook cases | Open; batch reruns every case and rewrites the manifest |
+| 12 | Run only new or unfinished workbook cases | Basic version done (run only new cases, flag unverified, dry run); fingerprints, retry of killed runs, orphans open |
 | 13 | Clean up the CIRRUS template deck | Partial; generated `BRINE` now matches salt tables; CIRRUS comparison and cleanup pending |
 
 ### 1. Compare scenario outputs automatically
@@ -635,11 +635,22 @@ zonation and aquifer-related transfer interpretation.
 
 ### 12. Run only new or unfinished workbook cases
 
-**Status: open; requested by the user (2026-10-12).** The batch runner
-processes every workbook scenario and rewrites `batch_manifest.json` with only
-a succeeded/failed status. After adding cases to a workbook, there is no safe
-way to start only the new ones: rerunning regenerates existing cases, can
-overwrite manual edits, and may resubmit jobs already queued or running.
+**Status: basic version implemented (2026-10-20); the rest is open.**
+Requested by the user (2026-10-12). Before this, the batch runner processed every
+workbook scenario, so after adding rows there was no safe way to start only the
+new ones.
+
+**Implemented** in `run_workbook_scenarios_batch.py`, using only files already in
+each case folder and no scheduler, so it works with or without LSF. Only cases
+without a folder are run. For an existing folder, `model/<stem>.LOG` ending with
+`Run completed` means finished (skipped); anything else is listed as unverified
+(not run) for the user to check, since the run may be live, killed or crashed.
+`--dry-run` prints the plan. A finished case whose `DesignMatrix` values differ
+from its saved `scenario.json` gets a warning. The manifest marks skipped cases
+`existing` or `unverified`; `--force` keeps its old meaning (rerun everything).
+Not done: a fingerprint covering `GridPolicy`, the well sheets and run options;
+automatic retry of killed runs; orphaned case folders; merging per-case history
+(job id, queue, timings) into the manifest.
 
 **In practice:** add rows to a workbook, ask which cases are new, changed,
 queued, running, finished, or failed, then prepare and submit only the cases

runscripts/run_workbook_scenarios_batch.py

@@ -29,6 +29,7 @@
 from pathlib import Path
 
 from src.GaP.libs.case_naming import resolve_case_stems, well_label
+from src.GaP.libs.models.simulation_scenario import SimulationDesign
 from src.WellClass.libs.utils import xlsx_to_simulation_design, xlsx_to_well_model
 
 
@@ -48,7 +49,8 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--cb-perm", type=float, default=0.05)
     parser.add_argument("--barrier-perm", type=float, default=0.05)
     parser.add_argument("--ali-way", action="store_true")
-    parser.add_argument("--force", action="store_true")
+    parser.add_argument("--force", action="store_true", help="Rerun every case, overwriting existing output; by default only cases without results are run.")
+    parser.add_argument("--dry-run", action="store_true", help="Print what would run for each case and stop.")
     parser.add_argument("--plot", action="store_true", help="Save a sketch and pressure QC plot for each scenario.")
     parser.add_argument(
         "--jobs",
@@ -124,7 +126,47 @@ def case_stems(design, xlsx_file: Path) -> dict[str, str]:
     return dict(zip(names, resolve_case_stems(well_label(xlsx_to_well_model(xlsx_file)), names)))
 
 
-def write_batch_manifest(design, args: argparse.Namespace, failed_cases: list[str]) -> Path:
+def run_completed(log: Path) -> bool:
+    """True when the CIRRUS log ends with its "Run completed" line."""
+    if not log.is_file():
+        return False
+    with log.open("rb") as stream:
+        stream.seek(max(0, log.stat().st_size - 4096))
+        return b"Run completed" in stream.read()
+
+
+def changed_inputs(case_root: Path, scenario) -> list[str]:
+    """DesignMatrix fields whose value differs from the scenario.json saved with the case."""
+    saved_path = case_root / "scenario.json"
+    if not saved_path.is_file():
+        return []
+    saved = json.loads(saved_path.read_text(encoding="utf-8"))
+    fields = type(scenario).model_fields
+    current = scenario.model_dump(mode="json")
+    return sorted(key for key, value in current.items() if key != "case_name" and saved.get(key, fields[key].default) != value)
+
+
+def plan_scenarios(design, args: argparse.Namespace) -> dict[str, tuple[str, str]]:
+    """Per case, ('run' | 'done' | 'verify', reason). Only cases without a folder are run."""
+    stems = case_stems(design, args.xlsx)
+    plan = {}
+    for scenario in design.scenarios:
+        stem = stems[scenario.case_name]
+        case_root = args.output_root / stem
+        if args.force or not case_root.exists():
+            plan[scenario.case_name] = ("run", "--force: rerun everything" if args.force else "new")
+        elif run_completed(case_root / "model" / f"{stem}.LOG"):
+            reason = "finished (model/<stem>.LOG ends with 'Run completed')"
+            changed = changed_inputs(case_root, scenario)
+            if changed:
+                reason += f"; WARNING: workbook row differs from the saved case in {', '.join(changed)}; delete its folder to rerun it"
+            plan[scenario.case_name] = ("done", reason)
+        else:
+            plan[scenario.case_name] = ("verify", "folder exists but its log has no 'Run completed' line: still running, killed or crashed?")
+    return plan
+
+
+def write_batch_manifest(design, args: argparse.Namespace, failed_cases: list[str], skipped: dict[str, str] | None = None) -> Path:
     stems = case_stems(design, args.xlsx)
     manifest = {
         "workbook": str(args.xlsx),
@@ -135,7 +177,7 @@ def write_batch_manifest(design, args: argparse.Namespace, failed_cases: list[st
                 "case_name": scenario.case_name,
                 "case_stem": stems[scenario.case_name],
                 "case_dir": stems[scenario.case_name],
-                "status": "failed" if scenario.case_name in failed_cases else "succeeded",
+                "status": "failed" if scenario.case_name in failed_cases else (skipped or {}).get(scenario.case_name, "succeeded"),
             }
             for index, scenario in enumerate(design.scenarios, 1)
         ],
@@ -208,15 +250,24 @@ def main() -> int:
     scenario_names = [scenario.case_name for scenario in design.scenarios]
     print(f"Found {len(scenario_names)} scenario(s): {', '.join(scenario_names)}")
 
-    successful_cases, failed_cases = run_scenarios(design, args)
-    manifest_path = write_batch_manifest(design, args, failed_cases)
+    plan = plan_scenarios(design, args)
+    for case_name, (action, reason) in plan.items():
+        print(f"  {action.upper():5s} {case_name}: {reason}")
+    if args.dry_run:
+        return 0
+    skipped = {name: "existing" if action == "done" else "unverified" for name, (action, _) in plan.items() if action != "run"}
+    to_run = SimulationDesign(scenarios=[s for s in design.scenarios if s.case_name not in skipped])
+    successful_cases, failed_cases = run_scenarios(to_run, args) if to_run.scenarios else ([], [])
+    manifest_path = write_batch_manifest(design, args, failed_cases, skipped)
     stems = case_stems(design, args.xlsx)
 
     print(f"\n{'='*70}")
     print("Batch execution summary:")
     print(f"  Total scenarios: {len(scenario_names)}")
     print(f"  Successful: {len(successful_cases)}")
     print(f"  Failed: {len(failed_cases)}")
+    print(f"  Already finished (skipped): {list(skipped.values()).count('existing')}")
+    print(f"  Need verification (not run): {list(skipped.values()).count('unverified')}")
     print(f"  Manifest: {manifest_path}")
 
     if successful_cases:
@@ -225,6 +276,12 @@ def main() -> int:
             case_root = args.output_root / stems[case_name]
             print(f"    - {case_name}: {case_root}")
 
+    unverified = [name for name, status in skipped.items() if status == "unverified"]
+    if unverified:
+        print("\n  NOT RUN: check whether these were killed or crashed (delete the case folder to rerun it):")
+        for case_name in unverified:
+            print(f"    - {case_name}: {args.output_root / stems[case_name]}")
+
     if failed_cases:
         print("\n  Failed cases:")
         for case_name in failed_cases:

src/GaP/README.md

@@ -109,6 +109,25 @@ now use 1.0, so regenerated wellbore cells differ slightly from older cases.
 Effective values and targeted overrides are recorded in `scenario.json`, and
 the resolved grid policy is saved in `grid_policy.json`.
 
+**Adding cases to a batch that has already run.** Run
+`run_workbook_scenarios_batch.py` again on the edited workbook and the same
+`--output-root`. Each workbook case is compared with the case folders there, and
+only cases without a folder are run. No scheduler is needed:
+
+| State | Evidence | Action |
+| --- | --- | --- |
+| new | no case folder | run |
+| finished | `model/<stem>.LOG` ends with `Run completed` | skip |
+| unverified | folder exists, but the log is missing or has no `Run completed` line | not run; listed for you to check |
+
+An unverified case may still be running, or may have been killed or crashed.
+Check it, then delete its folder to rerun it. `--dry-run` prints this plan and
+stops. If a finished case's `DesignMatrix` values differ from its saved
+`scenario.json`, the plan warns; delete its folder to rerun. Changes to
+`GridPolicy` or the well sheets are not detected. `--force` still reruns every
+case and overwrites its output. `batch_manifest.json` marks skipped cases
+`existing` or `unverified`.
+
 ### Experimental localized casing hole
 
 One optional hole per `DesignMatrix` row is supported. Supply all three fields

tests/gap/test_batch_scenarios.py

@@ -3,6 +3,7 @@
 from pathlib import Path
 import argparse
 import importlib.util
+import json
 import subprocess
 import sys
 import threading
@@ -266,3 +267,133 @@ def test_batch_rejects_non_positive_jobs(tmp_path):
 
     assert result.returncode == 2
     assert "--jobs must be at least 1" in result.stdout
+
+
+def _batch_args(tmp_path, workbook, **overrides):
+    return argparse.Namespace(
+        jobs=2, xlsx=workbook, output_root=tmp_path / "output", template_root=tmp_path, sim_command="fake",
+        run_final=False, simulation_years=1, start_date="2025-01-01", lgr_name="TEMP_LGR", oh_perm=None,
+        cb_perm=0.05, barrier_perm=0.05, ali_way=False, force=False, plot=False, dry_run=False, **overrides,
+    )
+
+
+def _existing_case(args, design, name, *, log=None, **saved):
+    stem = batch.case_stems(design, args.xlsx)[name]
+    root = args.output_root / stem
+    (root / "model").mkdir(parents=True)
+    scenario = design.select(name).model_dump(mode="json")
+    (root / "scenario.json").write_text(json.dumps({**scenario, **saved}))
+    if log is not None:
+        (root / "model" / f"{stem}.LOG").write_text(log)
+    return root
+
+
+DONE = " Time step 100\n Run completed, wall clock time =  4.1866E+03 s,  6.9776E+01 min\n"
+
+
+def test_plan_runs_only_cases_without_a_folder(tmp_path):
+    workbook = tmp_path / "multi_scenario.xlsx"
+    _write_multi_scenario_workbook(workbook, num_scenarios=5)
+    design = xlsx_to_simulation_design(workbook)
+    args = _batch_args(tmp_path, workbook)
+    _existing_case(args, design, "case_0", log=DONE)
+    _existing_case(args, design, "case_1", log=" Time step 40\n")  # killed: no final line
+    _existing_case(args, design, "case_2")  # staged, never produced a log
+    _existing_case(args, design, "case_3", log=DONE)
+
+    plan = batch.plan_scenarios(design, args)
+
+    assert {name: action for name, (action, _) in plan.items()} == {
+        "case_0": "done", "case_1": "verify", "case_2": "verify", "case_3": "done", "case_4": "run",
+    }
+    assert "killed or crashed" in plan["case_1"][1]
+    args.force = True
+    assert {action for action, _ in batch.plan_scenarios(design, args).values()} == {"run"}
+
+
+def test_completion_line_must_be_at_the_end_of_the_log(tmp_path):
+    log = tmp_path / "case.LOG"
+    log.write_text(DONE + "x" * 10000)
+    assert not batch.run_completed(log)
+    log.write_text("x" * 10000 + DONE)
+    assert batch.run_completed(log)
+    log.write_text(DONE)
+    assert batch.run_completed(log)
+    assert not batch.run_completed(tmp_path / "missing.LOG")
+
+
+def test_plan_warns_when_a_row_changed_since_it_finished(tmp_path):
+    workbook = tmp_path / "multi_scenario.xlsx"
+    _write_multi_scenario_workbook(workbook, num_scenarios=2)
+    design = xlsx_to_simulation_design(workbook)
+    args = _batch_args(tmp_path, workbook)
+    _existing_case(args, design, "case_0", log=DONE, cb_perm=5.0, z_fluid_contact=1.0)
+    _existing_case(args, design, "case_1", log=DONE)
+
+    plan = batch.plan_scenarios(design, args)
+
+    assert "WARNING" in plan["case_0"][1] and "cb_perm, z_fluid_contact" in plan["case_0"][1]
+    assert "WARNING" not in plan["case_1"][1]
+
+
+def test_older_saved_case_without_newer_fields_is_not_reported_as_changed(tmp_path):
+    workbook = tmp_path / "multi_scenario.xlsx"
+    _write_multi_scenario_workbook(workbook, num_scenarios=1)
+    design = xlsx_to_simulation_design(workbook)
+    args = _batch_args(tmp_path, workbook)
+    root = _existing_case(args, design, "case_0", log=DONE)
+    saved = json.loads((root / "scenario.json").read_text())
+    for newer in ("casing_hole_open", "free_fluid_permx", "free_fluid_poro"):
+        saved.pop(newer, None)
+    (root / "scenario.json").write_text(json.dumps(saved))
+
+    assert batch.changed_inputs(root, design.scenarios[0]) == []
+
+
+def test_main_runs_only_new_cases_and_lists_those_to_verify(tmp_path, monkeypatch, capsys):
+    workbook = tmp_path / "multi_scenario.xlsx"
+    _write_multi_scenario_workbook(workbook, num_scenarios=3)
+    design = xlsx_to_simulation_design(workbook)
+    args = _batch_args(tmp_path, workbook)
+    kept = _existing_case(args, design, "case_0", log=DONE)
+    _existing_case(args, design, "case_1", log=" Time step 40\n")
+    calls = []
+
+    def fake_run_scenario_case(_xlsx, case_name, *_rest):
+        calls.append(case_name)
+        return 0
+
+    monkeypatch.setattr(batch, "run_scenario_case", fake_run_scenario_case)
+    monkeypatch.setattr("sys.argv", [
+        "batch", "--xlsx", str(workbook), "--output-root", str(args.output_root), "--sim-command", "fake {deck}",
+    ])
+    assert batch.main() == 0
+
+    assert calls == ["case_2"]
+    manifest = json.loads((args.output_root / "batch_manifest.json").read_text())
+    assert {case["case_name"]: case["status"] for case in manifest["cases"]} == {
+        "case_0": "existing", "case_1": "unverified", "case_2": "succeeded",
+    }
+    assert (kept / "model").is_dir()
+    out = capsys.readouterr().out
+    assert "Already finished (skipped): 1" in out and "Need verification (not run): 1" in out
+    assert "NOT RUN: check whether these were killed or crashed" in out
+    assert "- case_1:" in out.split("NOT RUN")[1]
+
+
+def test_dry_run_prints_the_plan_and_changes_nothing(tmp_path, monkeypatch, capsys):
+    workbook = tmp_path / "multi_scenario.xlsx"
+    _write_multi_scenario_workbook(workbook, num_scenarios=3)
+    design = xlsx_to_simulation_design(workbook)
+    args = _batch_args(tmp_path, workbook)
+    _existing_case(args, design, "case_0", log=DONE)
+    _existing_case(args, design, "case_1")
+    monkeypatch.setattr(batch, "run_scenario_case", lambda *_: pytest.fail("dry run must not run cases"))
+    monkeypatch.setattr("sys.argv", [
+        "batch", "--xlsx", str(workbook), "--output-root", str(args.output_root), "--sim-command", "fake {deck}", "--dry-run",
+    ])
+    assert batch.main() == 0
+
+    out = capsys.readouterr().out
+    assert "DONE  case_0" in out and "VERIFY case_1" in out and "RUN   case_2: new" in out
+    assert not (args.output_root / "batch_manifest.json").exists()