run_incremental_compute.py

#!/usr/bin/env python3
"""
run_incremental_compute.py
--------------------------
Focused script that:
  1. Builds the three auxiliary libraries (rootsim_core_incremental, rng, topology)
     if not already built.
  2. Runs PHOLD incremental and PCS-0.48 incremental experiments IN PARALLEL
     (each experiment in its own subprocess, controlled by a process pool).
  3. Regenerates the three compute figures:
       - compute-plot_phold_err.pdf
       - compute-plot_pcs-0.48_err.pdf
       - compute-plot_tbc_err.pdf

TBC does NOT need new incremental data -- it only uses PRECISE/AUTONOMIC/MANUAL-A/MANUAL-B.
The TBC plot is regenerated from whatever data is already in the data/ directory.

Usage (on the Linux server, from the approximated_experiments directory):
    python3 run_incremental_compute.py

Output PDFs land in:
    ./data/results-renamed/compute/
"""
import json
import os
import sys
import subprocess
import shutil
import tempfile
from concurrent.futures import ProcessPoolExecutor, as_completed
from datetime import datetime

# ---------------------------------------------------------------------------
# Configuration  (keep in sync with run_experiments.py)
# ---------------------------------------------------------------------------
PHOLD_LPS    = 1024
PCS_LPS      = [1024]
PCS_END_GVT  = 2000
PCS_TA       = [0.48]

DATA_DIR     = "data"

# Root directory (absolute) — workers need this because they chdir into
# a private temp directory for each simulation run.
ROOT_DIR     = os.path.abspath(".")

# Maximum number of simulations to run in parallel.  Since each simulation
# uses many threads, keep this LOW (1–2) so threads don't oversubscribe.
# Auto-detected as max(1, cpu_count // 4) but never more than 4.
MAX_WORKERS  = 1

experiments_config = {}


# ---------------------------------------------------------------------------
# Configuration loader
# ---------------------------------------------------------------------------
def load_configuration():
    global experiments_config
    with open("experiments_config.json", "r", encoding="utf8") as f:
        experiments_config = json.load(f)
    print("Experiments configuration loaded")


# ---------------------------------------------------------------------------
# Submodule initialization
# ---------------------------------------------------------------------------
def init_submodules():
    """Make sure all git submodules are initialized and checked out.
    If a submodule directory has stale untracked files (e.g. from a previous
    partial setup), clean them first so git can checkout the right commit."""
    print("  Cleaning submodule directories of untracked files ...")
    # git clean -fd removes untracked files/dirs; safe even if submodule is empty
    for sub in ("rootsim_core", "rootsim_core_incremental", "rng", "topology"):
        os.system(f"git -C {sub} clean -fd 2>/dev/null || true")

    print("  Updating git submodules ...")
    r = os.system("git submodule update --init --recursive --force")
    if os.waitstatus_to_exitcode(r) != 0:
        print("git submodule update FAILED.\n"
              "Try manually:\n"
              "  git submodule foreach git clean -fd\n"
              "  git submodule update --init --recursive --force")
        sys.exit(1)
    print("  Submodules ready")



# ---------------------------------------------------------------------------
# Library builders with commit-stamp cache invalidation.
# All paths are ABSOLUTE (based on ROOT_DIR) to avoid any cwd-sensitivity.
# ---------------------------------------------------------------------------
def _git_head(repo_dir):
    """Return current HEAD SHA of a git repo/submodule, or '' on error."""
    r = subprocess.run(["git", "-C", repo_dir, "rev-parse", "HEAD"],
                       capture_output=True, text=True)
    return r.stdout.strip() if r.returncode == 0 else ""


def _lib_has_symbol(lib_path, symbol):
    """Return True if the static library exports `symbol`."""
    r = subprocess.run(["nm", lib_path], capture_output=True, text=True)
    return symbol in r.stdout


def _cmake_build(src_rel, build_rel, lib_rel, required_symbol, extra_flags=""):
    """Build a cmake project if needed.  Uses absolute paths throughout.

    Rebuilds if:
      - the library file doesn't exist, OR
      - the stored build-stamp SHA doesn't match the submodule HEAD, OR
      - the library is missing a required symbol (ultimate sanity check).
    """
    src   = os.path.join(ROOT_DIR, src_rel)
    build = os.path.join(ROOT_DIR, build_rel)
    lib   = os.path.join(ROOT_DIR, lib_rel)
    stamp = os.path.join(build, ".build_commit")

    current_sha = _git_head(src)

    need_rebuild = False
    reason = ""

    if not os.path.exists(lib):
        need_rebuild = True
        reason = "library not found"
    else:
        # Check commit stamp
        stored_sha = ""
        if os.path.exists(stamp):
            with open(stamp) as f:
                stored_sha = f.read().strip()
        if current_sha and stored_sha != current_sha:
            need_rebuild = True
            reason = (f"stale build "
                      f"(built from {stored_sha[:12] or '?'}, "
                      f"need {current_sha[:12] or '?'})")
        elif not _lib_has_symbol(lib, required_symbol):
            need_rebuild = True
            reason = f"library missing required symbol '{required_symbol}'"

    if not need_rebuild:
        print(f"  {lib_rel} up-to-date -- skipping")
        return

    print(f"  Rebuilding {lib_rel}: {reason}")
    if os.path.exists(build):
        shutil.rmtree(build)   # no ignore_errors — we want to know if this fails

    if not os.path.isdir(src) or not os.listdir(src):
        print(f"  ERROR: '{src_rel}' is empty/missing.\n"
              f"  Run:  git submodule update --init --recursive --force")
        sys.exit(1)

    r = os.system(f"cmake -S {src} -B {build} "
                  f"-DCMAKE_BUILD_TYPE=RELEASE {extra_flags}")
    if os.waitstatus_to_exitcode(r) != 0:
        print(f"cmake configure FAILED for {src_rel}")
        sys.exit(1)
    r = os.system(f"cmake --build {build}")
    if os.waitstatus_to_exitcode(r) != 0:
        print(f"cmake build FAILED for {src_rel}")
        sys.exit(1)
    if not os.path.exists(lib):
        print(f"cmake build finished but {lib_rel} not found!\n"
              f"  Files in build dir: {os.listdir(os.path.dirname(lib))}")
        sys.exit(1)
    if not _lib_has_symbol(lib, required_symbol):
        print(f"ERROR: built {lib_rel} but it still lacks '{required_symbol}'!\n"
              f"  The submodule may be at the wrong commit.\n"
              f"  Current submodule HEAD: {current_sha}")
        sys.exit(1)

    if current_sha:
        with open(stamp, "w") as f:
            f.write(current_sha + "\n")

    print(f"  {lib_rel} built OK (commit {current_sha[:12] if current_sha else '?'},"
          f" symbol '{required_symbol}' present)")


def prepare_libraries():
    # -- Approximated ROOT-Sim core (for PRECISE baseline runs) --
    _cmake_build("rootsim_core",
                 "rootsim_core_build",
                 "rootsim_core_build/src/librscore.a",
                 required_symbol="ScheduleNewEvent",
                 extra_flags="-DDISABLE_MPI=TRUE")
    # -- Incremental ROOT-Sim core --
    _cmake_build("rootsim_core_incremental",
                 "rootsim_core_incremental_build",
                 "rootsim_core_incremental_build/src/librscore.a",
                 required_symbol="WriteMemory",
                 extra_flags="-DWITH_MPI=OFF")
    _cmake_build("rng", "rng_build",
                 "rng_build/src/librsrng.a",
                 required_symbol="Random",
                 extra_flags="-DIMPORT_AS_LIB=ON")
    _cmake_build("topology", "topology_build",
                 "topology_build/src/librstopology.a",
                 required_symbol="GetReceiver",
                 extra_flags="-DIMPORT_AS_LIB=ON")




# ---------------------------------------------------------------------------
# config.h generator  (returns the file contents as a string)
# ---------------------------------------------------------------------------
def make_config(num_lps, num_threads, pcs_gvt=0, pcs_ta=0):
    """Config for INCREMENTAL experiments."""
    lines = [
        "#pragma once\n\n",
        f"#define NUM_THREADS {num_threads}\n",
        f"#define NUM_LPS {num_lps}\n",
        "#define SMART_RESTORE 0\n",
    ]
    if pcs_gvt != 0:
        lines += [
            f"#define PCS_END_GVT {pcs_gvt}\n",
            f"#define PCS_TA {pcs_ta}\n",
            f"#define PCS_STAT_FREQUENCY {pcs_ta}\n",
        ]
    return "".join(lines)


def make_config_precise(num_lps, num_threads, pcs_gvt=0, pcs_ta=0):
    """Config for PRECISE (approximated ROOT-Sim in precise mode) experiments."""
    lines = [
        "#pragma once\n\n",
        f"#define NUM_THREADS {num_threads}\n",
        f"#define NUM_LPS {num_lps}\n",
        "#define EXEC_MODE APPROXIMATED_MODE_PRECISE\n",
        "#define SMART_RESTORE 0\n",
        "#define APPROXIMATED_PERCENTAGE 1.0\n",
    ]
    if pcs_gvt != 0:
        lines += [
            f"#define PCS_END_GVT {pcs_gvt}\n",
            f"#define PCS_TA {pcs_ta}\n",
            f"#define PCS_STAT_FREQUENCY {pcs_ta}\n",
        ]
    return "".join(lines)


# ---------------------------------------------------------------------------
# Worker  (runs in a separate process so it is safe to compile in parallel)
# ---------------------------------------------------------------------------
def _run_one(model_folder, param_str, config_content):
    """Compile, run, and collect one simulation experiment.

    Returns (model_str, elapsed) on success, (model_str, -1) on failure.
    Uses a per-job temp header file (-include) instead of config.h so that
    multiple workers can compile simultaneously without racing on config.h.
    """
    model_str = f"{model_folder}_{param_str}"
    # Use absolute path — the worker will chdir into a private temp dir.
    root_dir  = ROOT_DIR
    bin_path  = os.path.join(root_dir, DATA_DIR, f"{model_str}.bin")

    if os.path.exists(bin_path):
        print(f"  [skip]  {model_str}")
        return model_str, 0.0

    print(f"  [start] {model_str}")

    # Write per-job config.h to a private temp directory.
    cfg_dir  = tempfile.mkdtemp(prefix=f"cfg_{model_str}_")
    # Each simulation runs in its OWN temp directory so that root_sir_stats.bin
    # and *.txt files never collide between parallel workers.
    run_dir  = tempfile.mkdtemp(prefix=f"run_{model_str}_")
    binary   = os.path.join(root_dir, f"model_{model_str}")

    try:
        with open(os.path.join(cfg_dir, "config.h"), "w") as cf:
            cf.write(config_content)

        compile_cmd = (
            f"gcc -O3 -Wno-incompatible-pointer-types "
            f"-I{cfg_dir} "
            f"-I{root_dir} "
            f"-I{root_dir}/rootsim_core_incremental/src "
            f"-I{root_dir}/rng/src/include "
            f"-I{root_dir}/topology/src/include "
            f"{root_dir}/{model_folder}/*.c "
            f"-Wl,--start-group "
            f"{root_dir}/rootsim_core_incremental_build/src/librscore.a "
            f"{root_dir}/rng_build/src/librsrng.a "
            f"{root_dir}/topology_build/src/librstopology.a "
            f"-Wl,--end-group "
            f"-lm -pthread -o {binary}"
        )
        r = subprocess.run(compile_cmd, shell=True, capture_output=True, text=True,
                           cwd=root_dir)
        if r.returncode != 0:
            print(f"  [ERROR] compile failed for {model_str}:\n{r.stderr[-2000:]}")
            return model_str, -1.0

        # Run the simulation inside its private temp directory.
        # root_sir_stats.bin lands in run_dir — no race with other workers.
        evo_file = os.path.join(run_dir, f"{model_str}_evo.txt")
        t0 = datetime.now()
        subprocess.run(f"{binary} | tee {evo_file}", shell=True, cwd=run_dir)
        elapsed = (datetime.now() - t0).total_seconds()

        os.makedirs(os.path.join(root_dir, DATA_DIR), exist_ok=True)

        stats_bin = os.path.join(run_dir, "root_sir_stats.bin")
        if not os.path.exists(stats_bin):
            print(f"  [ERROR] root_sir_stats.bin missing for {model_str} "
                  f"(simulation crashed? check {run_dir})")
            subprocess.run(f"rm -f {binary}", shell=True)
            return model_str, -1.0

        with open(os.path.join(root_dir, DATA_DIR, "times.txt"), "a") as timef:
            timef.write(f"{model_str.replace('_', chr(9))}\t{elapsed}\n")

        phases = os.path.join(run_dir, "root_sir_stats_phases.txt")
        if os.path.exists(phases):
            shutil.move(phases, os.path.join(root_dir, DATA_DIR,
                                             f"{model_str}_phases.txt"))
        shutil.move(stats_bin, bin_path)

        subprocess.run(
            f"python3 {root_dir}/rootsim_core_incremental/src/log/parse/"
            f"rootsim_stats.py {bin_path}",
            shell=True, cwd=root_dir)

        # Move any remaining .txt files (e.g. evo log)
        for fname in os.listdir(run_dir):
            if fname.endswith(".txt"):
                shutil.move(os.path.join(run_dir, fname),
                            os.path.join(root_dir, DATA_DIR, fname))

        subprocess.run(f"rm -f {binary}", shell=True)
        print(f"  [done]  {model_str}  ({elapsed:.1f}s)")
        return model_str, elapsed

    finally:
        shutil.rmtree(cfg_dir, ignore_errors=True)
        shutil.rmtree(run_dir, ignore_errors=True)


def _run_one_precise(model_folder, param_str, config_content):
    """Compile and run a PRECISE experiment using rootsim_core (approximated fork).

    Same structure as _run_one but links against rootsim_core instead of
    rootsim_core_incremental, and uses the rootsim_core stats parser.
    """
    model_str = f"{model_folder}_{param_str}"
    root_dir  = ROOT_DIR
    bin_path  = os.path.join(root_dir, DATA_DIR, f"{model_str}.bin")

    if os.path.exists(bin_path):
        print(f"  [skip]  {model_str}")
        return model_str, 0.0

    print(f"  [start] {model_str}")

    cfg_dir  = tempfile.mkdtemp(prefix=f"cfg_{model_str}_")
    run_dir  = tempfile.mkdtemp(prefix=f"run_{model_str}_")
    binary   = os.path.join(root_dir, f"model_{model_str}")

    try:
        with open(os.path.join(cfg_dir, "config.h"), "w") as cf:
            cf.write(config_content)

        compile_cmd = (
            f"gcc -O3 -Wno-incompatible-pointer-types "
            f"-I{cfg_dir} "
            f"-I{root_dir} "
            f"-I{root_dir}/rootsim_core/src "
            f"{root_dir}/{model_folder}/*.c "
            f"{root_dir}/rootsim_core_build/src/librscore.a "
            f"-lm -pthread -o {binary}"
        )
        r = subprocess.run(compile_cmd, shell=True, capture_output=True, text=True,
                           cwd=root_dir)
        if r.returncode != 0:
            print(f"  [ERROR] compile failed for {model_str}:\n{r.stderr[-2000:]}")
            return model_str, -1.0

        evo_file = os.path.join(run_dir, f"{model_str}_evo.txt")
        t0 = datetime.now()
        subprocess.run(f"{binary} | tee {evo_file}", shell=True, cwd=run_dir)
        elapsed = (datetime.now() - t0).total_seconds()

        os.makedirs(os.path.join(root_dir, DATA_DIR), exist_ok=True)

        stats_bin = os.path.join(run_dir, "root_sir_stats.bin")
        if not os.path.exists(stats_bin):
            print(f"  [ERROR] root_sir_stats.bin missing for {model_str} "
                  f"(simulation crashed? check {run_dir})")
            subprocess.run(f"rm -f {binary}", shell=True)
            return model_str, -1.0

        with open(os.path.join(root_dir, DATA_DIR, "times.txt"), "a") as timef:
            timef.write(f"{model_str.replace('_', chr(9))}\t{elapsed}\n")

        phases = os.path.join(run_dir, "root_sir_stats_phases.txt")
        if os.path.exists(phases):
            shutil.move(phases, os.path.join(root_dir, DATA_DIR,
                                             f"{model_str}_phases.txt"))
        shutil.move(stats_bin, bin_path)

        # Use rootsim_core parser (not incremental) for PRECISE .bin files
        subprocess.run(
            f"python3 {root_dir}/rootsim_core/src/log/parse/"
            f"rootsim_stats.py {bin_path}",
            shell=True, cwd=root_dir)

        for fname in os.listdir(run_dir):
            if fname.endswith(".txt"):
                shutil.move(os.path.join(run_dir, fname),
                            os.path.join(root_dir, DATA_DIR, fname))

        subprocess.run(f"rm -f {binary}", shell=True)
        print(f"  [done]  {model_str}  ({elapsed:.1f}s)")
        return model_str, elapsed

    finally:
        shutil.rmtree(cfg_dir, ignore_errors=True)
        shutil.rmtree(run_dir, ignore_errors=True)


# ---------------------------------------------------------------------------
# Job generators — INCREMENTAL
# ---------------------------------------------------------------------------
def phold_jobs():
    max_threads = os.cpu_count() or 1
    skipped = []
    for iteration in range(experiments_config["phold"]["repetitions"]):
        for num_threads in experiments_config["phold"]["threads"]:
            if num_threads > max_threads:
                skipped.append(num_threads)
                continue
            cfg = make_config(PHOLD_LPS, num_threads)
            param_str = f"{PHOLD_LPS}_{num_threads}_INCREMENTAL_{iteration}"
            yield "phold_incremental", param_str, cfg
    if skipped:
        print(f"  [WARN] PHOLD incr: skipping thread counts > {max_threads} (cpu_count): "
              f"{sorted(set(skipped))}")


def pcs_jobs():
    max_threads = os.cpu_count() or 1
    skipped = []
    for lp in PCS_LPS:
        for iteration in range(experiments_config["pcs"]["repetitions"]):
            for num_threads in experiments_config["pcs"]["threads"]:
                if num_threads > max_threads:
                    skipped.append(num_threads)
                    continue
                for p in PCS_TA:
                    cfg = make_config(lp, num_threads, pcs_gvt=PCS_END_GVT, pcs_ta=p)
                    param_str = f"INCREMENTAL_{p}_{lp}_{num_threads}_{iteration}"
                    yield "pcs_incremental", param_str, cfg
    if skipped:
        print(f"  [WARN] PCS incr: skipping thread counts > {max_threads} (cpu_count): "
              f"{sorted(set(skipped))}")


# ---------------------------------------------------------------------------
# Job generators — PRECISE baseline  (uses rootsim_core + phold/pcs models)
# ---------------------------------------------------------------------------
def phold_precise_jobs():
    max_threads = os.cpu_count() or 1
    skipped = []
    for iteration in range(experiments_config["phold"]["repetitions"]):
        for num_threads in experiments_config["phold"]["threads"]:
            if num_threads > max_threads:
                skipped.append(num_threads)
                continue
            cfg = make_config_precise(PHOLD_LPS, num_threads)
            param_str = f"{PHOLD_LPS}_{num_threads}_PRECISE_{iteration}"
            yield "phold", param_str, cfg
    if skipped:
        print(f"  [WARN] PHOLD precise: skipping thread counts > {max_threads}: "
              f"{sorted(set(skipped))}")


def pcs_precise_jobs():
    max_threads = os.cpu_count() or 1
    skipped = []
    for lp in PCS_LPS:
        for iteration in range(experiments_config["pcs"]["repetitions"]):
            for num_threads in experiments_config["pcs"]["threads"]:
                if num_threads > max_threads:
                    skipped.append(num_threads)
                    continue
                for p in PCS_TA:
                    cfg = make_config_precise(lp, num_threads,
                                             pcs_gvt=PCS_END_GVT, pcs_ta=p)
                    param_str = f"PRECISE_{p}_{lp}_{num_threads}_{iteration}"
                    yield "pcs", param_str, cfg
    if skipped:
        print(f"  [WARN] PCS precise: skipping thread counts > {max_threads}: "
              f"{sorted(set(skipped))}")


# ---------------------------------------------------------------------------
# Parallel runner
# ---------------------------------------------------------------------------
def run_parallel(jobs, worker_fn=_run_one):
    """Run jobs in parallel using the specified worker function."""
    os.makedirs(DATA_DIR, exist_ok=True)
    jobs = list(jobs)
    print(f"Submitting {len(jobs)} jobs (MAX_WORKERS={MAX_WORKERS}) ...")
    failed = []
    with ProcessPoolExecutor(max_workers=MAX_WORKERS) as pool:
        futures = {
            pool.submit(worker_fn, mf, ps, cfg): f"{mf}_{ps}"
            for mf, ps, cfg in jobs
        }
        for fut in as_completed(futures):
            label = futures[fut]
            try:
                _, elapsed = fut.result()
                if elapsed < 0:
                    failed.append(label)
            except Exception as exc:
                print(f"  [EXCEPTION] {label}: {exc}")
                failed.append(label)

    if failed:
        print(f"\nFAILED ({len(failed)}):")
        for f in failed:
            print(f"  {f}")
        print("You may rerun this script -- successful runs are cached.")
    else:
        print("All jobs finished successfully.")


# ---------------------------------------------------------------------------
# Plot generation
# ---------------------------------------------------------------------------
def generate_plots():
    import matplotlib
    matplotlib.use("Agg")   # headless / no X display needed

    # Make sure the local packages are on the path
    for p in [".", "plot_scripts"]:
        if p not in sys.path:
            sys.path.insert(0, p)

    from plot_scripts.sme import load_rs_data, plot_model
    import matplotlib.pyplot as plt

    plt.rcParams["font.family"] = ["sans"]
    plt.rcParams["axes.unicode_minus"] = False

    print(f"Loading stats from {DATA_DIR}/ ...")
    threads, data = load_rs_data(DATA_DIR)

    target_dir = os.path.join(DATA_DIR, "results-renamed", "compute")
    os.makedirs(target_dir, exist_ok=True)

    targets = [
        ("phold",    "compute-plot_phold_err.pdf"),
        ("pcs-0.48", "compute-plot_pcs-0.48_err.pdf"),
        ("tbc",      "compute-plot_tbc_err.pdf"),
    ]
    for model_name, out_name in targets:
        # Check we have at least some data for this model before plotting
        has_data = any(k[0] == model_name for k in data)
        if not has_data:
            print(f"  [WARN] No data found for model '{model_name}' -- skipping plot")
            continue
        print(f"  plotting {model_name}  ->  {out_name}")
        plot_model(model_name, threads, data)
        src = f"plot_{model_name}.pdf"
        dst = os.path.join(target_dir, out_name)
        if os.path.exists(src):
            shutil.move(src, dst)
            print(f"           saved to {dst}")
        else:
            print(f"  [WARN] {src} was not produced")


# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------
if __name__ == "__main__":
    load_configuration()

    print("\n=== Initializing git submodules ===")
    init_submodules()

    print("\n=== Building libraries ===")
    prepare_libraries()


    print("\n=== Running PRECISE baseline experiments (parallel) ===")
    run_parallel(list(phold_precise_jobs()) + list(pcs_precise_jobs()),
                 worker_fn=_run_one_precise)

    print("\n=== Running PHOLD + PCS incremental experiments (parallel) ===")
    run_parallel(list(phold_jobs()) + list(pcs_jobs()),
                 worker_fn=_run_one)

    print("\n=== Generating compute plots ===")
    generate_plots()

    print("\nDone.  PDFs written to data/results-renamed/compute/")