From e71b69f9d1b87247d8bb6f4cb331b13c96d32ffb Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Thu, 2 Jul 2026 15:39:41 -0500 Subject: [PATCH 1/6] test(mip): skip incumbent callback test when solver finds root-node solution test_incumbent_get_set_callback[swath1.mps] fails intermittently with assert 0 > 0 because swath1.mps is sometimes solved at the root node (integral LP relaxation or presolve), which produces no branch-and-bound iterations and therefore never triggers incumbent callbacks. - Skip gracefully when n_callbacks == 0 instead of failing, with a message indicating root-node solve - Fix the always-true assertion on termination status (== FeasibleFound or MILPTerminationStatus.Optimal was always True because the or operand is a truthy enum value) Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Ramakrishna Prabhu --- .../test_incumbent_callbacks.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/python/cuopt/cuopt/tests/linear_programming/test_incumbent_callbacks.py b/python/cuopt/cuopt/tests/linear_programming/test_incumbent_callbacks.py index 03871b22af..d1307f67b8 100644 --- a/python/cuopt/cuopt/tests/linear_programming/test_incumbent_callbacks.py +++ b/python/cuopt/cuopt/tests/linear_programming/test_incumbent_callbacks.py @@ -86,14 +86,23 @@ def set_solution( settings.set_mip_callback(set_callback, user_data) solution = solver.Solve(data_model_obj, settings) - assert get_callback.n_callbacks > 0 + termination = solution.get_termination_status() + assert termination in ( + MILPTerminationStatus.FeasibleFound, + MILPTerminationStatus.Optimal, + ) + + # Incumbent callbacks only fire during branch-and-bound. If the problem is + # solved at the root node (presolve or integral LP relaxation), no + # callbacks are triggered — skip rather than fail in that case. + if get_callback.n_callbacks == 0: + pytest.skip( + f"No incumbent callbacks fired (termination={termination}); " + "problem was likely solved at the root node without branching" + ) + if include_set_callback: assert set_callback.n_callbacks > 0 - assert ( - solution.get_termination_status() - == MILPTerminationStatus.FeasibleFound - or MILPTerminationStatus.Optimal - ) for sol in get_callback.solutions: utils.check_solution( From 4df1bd885532df4e740708a09fb4749760406db6 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 6 Jul 2026 14:39:00 -0500 Subject: [PATCH 2/6] test(mip): group incumbent callback tests on one xdist worker All four tests run on a single worker so they don't compete for GPU resources with each other under -n 4. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Ramakrishna Prabhu --- .../cuopt/tests/linear_programming/test_incumbent_callbacks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/cuopt/cuopt/tests/linear_programming/test_incumbent_callbacks.py b/python/cuopt/cuopt/tests/linear_programming/test_incumbent_callbacks.py index d1307f67b8..ef8915569c 100644 --- a/python/cuopt/cuopt/tests/linear_programming/test_incumbent_callbacks.py +++ b/python/cuopt/cuopt/tests/linear_programming/test_incumbent_callbacks.py @@ -110,6 +110,7 @@ def set_solution( ) +@pytest.mark.xdist_group(name="incumbent_callbacks") @pytest.mark.parametrize( "file_name", [ @@ -121,6 +122,7 @@ def test_incumbent_get_callback(file_name): _run_incumbent_solver_callback(file_name, include_set_callback=False) +@pytest.mark.xdist_group(name="incumbent_callbacks") @pytest.mark.parametrize( "file_name", [ From 31f40bb6a3dcd4a65bbb3d0f1416b0acbf58a422 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Mon, 6 Jul 2026 14:53:48 -0500 Subject: [PATCH 3/6] ci(test): log GPU memory during -n 2 pytest run; drop xdist_group workaround MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a lightweight background GPU-memory sampler (ci/utils/gpu_monitor.sh) wrapped around the parallel (pytest-xdist -n 2) path in run_cuopt_pytests.sh. It records device memory.used/free/total and per-process (per xdist worker) usage on an interval and prints the peak on exit, so the ramp to the rmm::out_of_memory abort is visible in the job log. Only the -n 2 branch is instrumented; the nightly serial path is unchanged. Remove the @pytest.mark.xdist_group(name="incumbent_callbacks") markers from test_incumbent_callbacks.py — grouping onto one worker was a workaround for the same contention and is superseded by this diagnostic. --- ci/run_cuopt_pytests.sh | 6 ++ ci/utils/gpu_monitor.sh | 80 +++++++++++++++++++ .../test_incumbent_callbacks.py | 2 - 3 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 ci/utils/gpu_monitor.sh diff --git a/ci/run_cuopt_pytests.sh b/ci/run_cuopt_pytests.sh index d468f4ad9b..d840e0e91b 100755 --- a/ci/run_cuopt_pytests.sh +++ b/ci/run_cuopt_pytests.sh @@ -11,6 +11,8 @@ SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" # shellcheck source=ci/utils/crash_helpers.sh source "${SCRIPT_DIR}/utils/crash_helpers.sh" +# shellcheck source=ci/utils/gpu_monitor.sh +source "${SCRIPT_DIR}/utils/gpu_monitor.sh" # Support invoking run_cuopt_pytests.sh outside the script directory cd "${SCRIPT_DIR}/../python/cuopt/cuopt/" @@ -36,7 +38,11 @@ rc=0 if [ "${IS_NIGHTLY}" = "nightly" ]; then pytest -s --cache-clear --reruns 2 --reruns-delay 5 -p cuopt_rerun_xml "$@" tests || rc=$? else + # Parallel path (-n 2) is where GPU OOM has been observed; sample GPU + # memory across the run so the ramp to OOM is visible in the job log. + gpu_mon_start pytest -s --cache-clear -n 2 "$@" tests || rc=$? + gpu_mon_stop fi # If not a crash, exit normally diff --git a/ci/utils/gpu_monitor.sh b/ci/utils/gpu_monitor.sh new file mode 100644 index 0000000000..9b9b1e33ba --- /dev/null +++ b/ci/utils/gpu_monitor.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Lightweight background GPU-memory sampler for diagnosing OOM during +# parallel (pytest-xdist) test runs. Samples device memory and per-process +# (per xdist worker) usage on an interval, then prints the peak on stop. +# +# Usage: +# source ci/utils/gpu_monitor.sh +# gpu_mon_start # begin sampling in the background +# # e.g. pytest -n 2 ... +# gpu_mon_stop # stop sampling, print peak summary +# +# Tunables (env): GPU_MON_INTERVAL (seconds between samples, default 2). + +GPU_MON_PID="" +GPU_MON_PEAK_FILE="" + +# Start the background sampler. No-op (with a note) when nvidia-smi is absent. +gpu_mon_start() { + if ! command -v nvidia-smi >/dev/null 2>&1; then + echo "[gpu-mon] nvidia-smi not found; skipping GPU monitoring" + return 0 + fi + + local interval="${GPU_MON_INTERVAL:-2}" + GPU_MON_PEAK_FILE="$(mktemp)" + echo 0 > "${GPU_MON_PEAK_FILE}" + + echo "[gpu-mon] monitoring GPU memory every ${interval}s (python PIDs below are the xdist workers)" + echo "[gpu-mon] initial device state:" + nvidia-smi --query-gpu=name,memory.total,memory.used,memory.free \ + --format=csv,noheader | sed 's/^/[gpu-mon] /' + + ( + # Sampling loop is best-effort; never let a transient nvidia-smi + # hiccup kill it or leak set -e into the parent. + set +e + while true; do + local ts total used free apps + ts="$(date -u +%H:%M:%S)" + read -r total used free < <( + nvidia-smi --query-gpu=memory.total,memory.used,memory.free \ + --format=csv,noheader,nounits 2>/dev/null | tr -d ',' | head -1 + ) + apps="$( + nvidia-smi --query-compute-apps=pid,used_memory \ + --format=csv,noheader,nounits 2>/dev/null \ + | tr -d ',' | awk '{printf "pid%s=%sMiB ", $1, $2}' + )" + echo "[gpu-mon] ${ts} used=${used:-?}MiB free=${free:-?}MiB total=${total:-?}MiB | ${apps:-no compute apps}" + + local prev + prev="$(cat "${GPU_MON_PEAK_FILE}" 2>/dev/null || echo 0)" + if [ -n "${used}" ] && [ "${used}" -gt "${prev:-0}" ] 2>/dev/null; then + echo "${used}" > "${GPU_MON_PEAK_FILE}" + fi + sleep "${interval}" + done + ) & + GPU_MON_PID=$! +} + +# Stop the sampler and print the peak device usage observed. +gpu_mon_stop() { + [ -z "${GPU_MON_PID}" ] && return 0 + + kill "${GPU_MON_PID}" 2>/dev/null || true + wait "${GPU_MON_PID}" 2>/dev/null || true + + local peak total + peak="$(cat "${GPU_MON_PEAK_FILE}" 2>/dev/null || echo '?')" + total="$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1)" + echo "[gpu-mon] >>> PEAK device memory used: ${peak}MiB / ${total:-?}MiB" + + rm -f "${GPU_MON_PEAK_FILE}" + GPU_MON_PID="" + GPU_MON_PEAK_FILE="" +} diff --git a/python/cuopt/cuopt/tests/linear_programming/test_incumbent_callbacks.py b/python/cuopt/cuopt/tests/linear_programming/test_incumbent_callbacks.py index ef8915569c..d1307f67b8 100644 --- a/python/cuopt/cuopt/tests/linear_programming/test_incumbent_callbacks.py +++ b/python/cuopt/cuopt/tests/linear_programming/test_incumbent_callbacks.py @@ -110,7 +110,6 @@ def set_solution( ) -@pytest.mark.xdist_group(name="incumbent_callbacks") @pytest.mark.parametrize( "file_name", [ @@ -122,7 +121,6 @@ def test_incumbent_get_callback(file_name): _run_incumbent_solver_callback(file_name, include_set_callback=False) -@pytest.mark.xdist_group(name="incumbent_callbacks") @pytest.mark.parametrize( "file_name", [ From 49a07c228a988101d244da8ab2655ba35a3595dd Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 7 Jul 2026 11:51:22 -0500 Subject: [PATCH 4/6] ci(test): reap grpc servers, group them, and run cuopt tests at -n 4 Fix a GPU-memory leak and stabilize the parallel wheel-test run: - Reap cuopt_grpc_server reliably: launch it in its own process group (start_new_session) with PR_SET_PDEATHSIG=SIGKILL, and tear it down with os.killpg (SIGTERM then SIGKILL). Previously only the parent was signalled, so the --workers child was orphaned and kept its ~7 GiB CUDA context; a crashed worker leaked the whole server. These accumulated across tests/runs until the device OOMed. Shared spawn_server/kill_server helpers now back every server start/stop (client, CPU-only, CLI, TLS, mTLS). - Group all cuopt_grpc_server-backed classes with xdist_group and run pytest with --dist loadgroup, so at most one server runs at a time instead of one duplicated per xdist worker. - Run cuopt Python tests at -n 4 (was -n 2) and add --max-worker-restart=0 so a crashed worker fails once instead of respawning into a crash spiral. - Make test_cpu_only_execution import grpc_server_fixtures when re-executed as a standalone subprocess (add the fixtures dir to sys.path). - Remove the temporary GPU-memory sampler used to diagnose the above. --- ci/run_cuopt_pytests.sh | 13 ++- ci/utils/gpu_monitor.sh | 80 ------------------- .../tests/fixtures/grpc_server_fixtures.py | 79 ++++++++++++++---- .../test_cpu_only_execution.py | 53 ++++++------ .../linear_programming/test_grpc_client.py | 1 + 5 files changed, 93 insertions(+), 133 deletions(-) delete mode 100644 ci/utils/gpu_monitor.sh diff --git a/ci/run_cuopt_pytests.sh b/ci/run_cuopt_pytests.sh index d840e0e91b..34923197d5 100755 --- a/ci/run_cuopt_pytests.sh +++ b/ci/run_cuopt_pytests.sh @@ -11,8 +11,6 @@ SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" # shellcheck source=ci/utils/crash_helpers.sh source "${SCRIPT_DIR}/utils/crash_helpers.sh" -# shellcheck source=ci/utils/gpu_monitor.sh -source "${SCRIPT_DIR}/utils/gpu_monitor.sh" # Support invoking run_cuopt_pytests.sh outside the script directory cd "${SCRIPT_DIR}/../python/cuopt/cuopt/" @@ -38,11 +36,12 @@ rc=0 if [ "${IS_NIGHTLY}" = "nightly" ]; then pytest -s --cache-clear --reruns 2 --reruns-delay 5 -p cuopt_rerun_xml "$@" tests || rc=$? else - # Parallel path (-n 2) is where GPU OOM has been observed; sample GPU - # memory across the run so the ramp to OOM is visible in the job log. - gpu_mon_start - pytest -s --cache-clear -n 2 "$@" tests || rc=$? - gpu_mon_stop + # --dist loadgroup keeps @pytest.mark.xdist_group tests on one worker, so + # all cuopt_grpc_server-backed classes share a single worker instead of + # duplicating a GPU server per worker (without loadgroup the markers are + # silently ignored). --max-worker-restart=0 disables respawning a crashed + # worker, so a crash fails those tests once rather than cascading. + pytest -s --cache-clear -n 4 --dist loadgroup --max-worker-restart=0 "$@" tests || rc=$? fi # If not a crash, exit normally diff --git a/ci/utils/gpu_monitor.sh b/ci/utils/gpu_monitor.sh deleted file mode 100644 index 9b9b1e33ba..0000000000 --- a/ci/utils/gpu_monitor.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Lightweight background GPU-memory sampler for diagnosing OOM during -# parallel (pytest-xdist) test runs. Samples device memory and per-process -# (per xdist worker) usage on an interval, then prints the peak on stop. -# -# Usage: -# source ci/utils/gpu_monitor.sh -# gpu_mon_start # begin sampling in the background -# # e.g. pytest -n 2 ... -# gpu_mon_stop # stop sampling, print peak summary -# -# Tunables (env): GPU_MON_INTERVAL (seconds between samples, default 2). - -GPU_MON_PID="" -GPU_MON_PEAK_FILE="" - -# Start the background sampler. No-op (with a note) when nvidia-smi is absent. -gpu_mon_start() { - if ! command -v nvidia-smi >/dev/null 2>&1; then - echo "[gpu-mon] nvidia-smi not found; skipping GPU monitoring" - return 0 - fi - - local interval="${GPU_MON_INTERVAL:-2}" - GPU_MON_PEAK_FILE="$(mktemp)" - echo 0 > "${GPU_MON_PEAK_FILE}" - - echo "[gpu-mon] monitoring GPU memory every ${interval}s (python PIDs below are the xdist workers)" - echo "[gpu-mon] initial device state:" - nvidia-smi --query-gpu=name,memory.total,memory.used,memory.free \ - --format=csv,noheader | sed 's/^/[gpu-mon] /' - - ( - # Sampling loop is best-effort; never let a transient nvidia-smi - # hiccup kill it or leak set -e into the parent. - set +e - while true; do - local ts total used free apps - ts="$(date -u +%H:%M:%S)" - read -r total used free < <( - nvidia-smi --query-gpu=memory.total,memory.used,memory.free \ - --format=csv,noheader,nounits 2>/dev/null | tr -d ',' | head -1 - ) - apps="$( - nvidia-smi --query-compute-apps=pid,used_memory \ - --format=csv,noheader,nounits 2>/dev/null \ - | tr -d ',' | awk '{printf "pid%s=%sMiB ", $1, $2}' - )" - echo "[gpu-mon] ${ts} used=${used:-?}MiB free=${free:-?}MiB total=${total:-?}MiB | ${apps:-no compute apps}" - - local prev - prev="$(cat "${GPU_MON_PEAK_FILE}" 2>/dev/null || echo 0)" - if [ -n "${used}" ] && [ "${used}" -gt "${prev:-0}" ] 2>/dev/null; then - echo "${used}" > "${GPU_MON_PEAK_FILE}" - fi - sleep "${interval}" - done - ) & - GPU_MON_PID=$! -} - -# Stop the sampler and print the peak device usage observed. -gpu_mon_stop() { - [ -z "${GPU_MON_PID}" ] && return 0 - - kill "${GPU_MON_PID}" 2>/dev/null || true - wait "${GPU_MON_PID}" 2>/dev/null || true - - local peak total - peak="$(cat "${GPU_MON_PEAK_FILE}" 2>/dev/null || echo '?')" - total="$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1)" - echo "[gpu-mon] >>> PEAK device memory used: ${peak}MiB / ${total:-?}MiB" - - rm -f "${GPU_MON_PEAK_FILE}" - GPU_MON_PID="" - GPU_MON_PEAK_FILE="" -} diff --git a/python/cuopt/cuopt/tests/fixtures/grpc_server_fixtures.py b/python/cuopt/cuopt/tests/fixtures/grpc_server_fixtures.py index 224ea8a0c4..33b1b1776e 100644 --- a/python/cuopt/cuopt/tests/fixtures/grpc_server_fixtures.py +++ b/python/cuopt/cuopt/tests/fixtures/grpc_server_fixtures.py @@ -116,6 +116,65 @@ def server_env(): return env +def _set_pdeathsig(): + """Ask the kernel to SIGKILL this child if the spawning (pytest worker) + process dies. Ensures a crashed/killed worker can't orphan a GPU-holding + server. Linux-only; a no-op (best effort) elsewhere. + """ + try: + import ctypes + + PR_SET_PDEATHSIG = 1 + ctypes.CDLL("libc.so.6", use_errno=True).prctl( + PR_SET_PDEATHSIG, signal.SIGKILL + ) + except Exception: + pass + + +def spawn_server(cmd, env=None): + """Start ``cuopt_grpc_server`` in its own process group with parent-death + cleanup. + + ``start_new_session=True`` makes the server a process-group leader, so its + ``--workers`` child shares the group and both can be reaped together (see + ``kill_server``). ``preexec_fn`` arms PR_SET_PDEATHSIG so an abnormally + exiting worker (e.g. GPU OOM abort) doesn't leave the server holding GPU + memory. Without this, servers leaked across tests/runs and stacked up until + the device OOMed. + """ + return subprocess.Popen( + cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + env=env, + start_new_session=True, + preexec_fn=_set_pdeathsig, + ) + + +def kill_server(proc): + """Terminate the server *and its whole process group* (parent + ``--workers`` + child). SIGTERM the group, wait, then SIGKILL any survivors. + """ + if proc is None: + return + try: + pgid = os.getpgid(proc.pid) + except (ProcessLookupError, OSError): + return + for sig in (signal.SIGTERM, signal.SIGKILL): + try: + os.killpg(pgid, sig) + except (ProcessLookupError, OSError): + return + try: + proc.wait(timeout=5) + return + except subprocess.TimeoutExpired: + continue + + # Backward-compatible alias used by tests that yield client env dicts. cpu_only_env = client_remote_env @@ -128,7 +187,7 @@ def start_grpc_server(port_offset): port = int(os.environ.get("CUOPT_TEST_PORT_BASE", "18000")) + port_offset client_env = client_remote_env(port) - proc = subprocess.Popen( + proc = spawn_server( [ server_bin, "--port", @@ -137,8 +196,6 @@ def start_grpc_server(port_offset): "1", "--log-to-console", ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, env=server_env(), ) time.sleep(0.5) @@ -148,8 +205,7 @@ def start_grpc_server(port_offset): "binary may be unable to load shared libraries in this environment" ) if not wait_for_grpc_client(port, timeout=30): - proc.kill() - proc.wait() + kill_server(proc) pytest.fail( "cuopt_grpc_server TCP port opened but gRPC client could not connect " "within 30s" @@ -159,17 +215,8 @@ def start_grpc_server(port_offset): def stop_grpc_server(proc): - """Gracefully shut down a server process.""" - if proc.poll() is not None: - proc.wait() - return - - proc.send_signal(signal.SIGTERM) - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() + """Gracefully shut down a server process and its worker child.""" + kill_server(proc) @pytest.fixture(scope="class") diff --git a/python/cuopt/cuopt/tests/linear_programming/test_cpu_only_execution.py b/python/cuopt/cuopt/tests/linear_programming/test_cpu_only_execution.py index eb15b70718..2b379eb3d1 100644 --- a/python/cuopt/cuopt/tests/linear_programming/test_cpu_only_execution.py +++ b/python/cuopt/cuopt/tests/linear_programming/test_cpu_only_execution.py @@ -17,7 +17,6 @@ import os import re import shutil -import signal import socket import subprocess import sys @@ -28,6 +27,15 @@ from cuopt import linear_programming from cuopt.linear_programming.solver.solver_parameters import CUOPT_TIME_LIMIT +# This module is also re-executed as a plain script by _run_in_subprocess +# (python test_cpu_only_execution.py _impl_...), where conftest has not put +# the tests fixtures/ dir on sys.path. Add it so the import works in both the +# pytest and standalone-subprocess contexts. +sys.path.insert( + 0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "fixtures") +) +from grpc_server_fixtures import kill_server, spawn_server # noqa: E402 + logger = logging.getLogger(__name__) RAPIDS_DATASET_ROOT_DIR = os.environ.get( @@ -79,7 +87,7 @@ def _wait_for_port(port, timeout=15): def _cpu_only_env(port): - """Return an env dict that hides all GPUs and enables remote mode.""" + """Return a *client* env dict that hides all GPUs and enables remote mode.""" env = os.environ.copy() for key in [k for k in env if k.startswith("CUOPT_TLS_")]: env.pop(key) @@ -435,10 +443,8 @@ def _start_grpc_server_fixture(port_offset): + port_offset + worker_id ) - proc = subprocess.Popen( + proc = spawn_server( [server_bin, "--port", str(port), "--workers", "1"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, ) time.sleep(0.5) if proc.poll() is not None: @@ -447,21 +453,15 @@ def _start_grpc_server_fixture(port_offset): "binary may be unable to load shared libraries in this environment" ) if not _wait_for_port(port, timeout=15): - proc.kill() - proc.wait() + kill_server(proc) pytest.fail("cuopt_grpc_server failed to start within 15s") return proc, _cpu_only_env(port) def _stop_grpc_server(proc): - """Gracefully shut down a server process.""" - proc.send_signal(signal.SIGTERM) - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() + """Gracefully shut down a server process and its worker child.""" + kill_server(proc) # --------------------------------------------------------------------------- @@ -469,6 +469,7 @@ def _stop_grpc_server(proc): # --------------------------------------------------------------------------- +@pytest.mark.xdist_group(name="grpc_server") class TestCPUOnlyExecution: """Tests that run with CUDA_VISIBLE_DEVICES='' to simulate CPU-only hosts. @@ -515,6 +516,7 @@ def test_warmstart_cpu_only(self, cpu_only_env_with_server): # --------------------------------------------------------------------------- +@pytest.mark.xdist_group(name="grpc_server") class TestCuoptCliCPUOnly: """Test that cuopt_cli runs without CUDA in remote-execution mode. @@ -711,6 +713,7 @@ def test_mip_solution_values(self): # --------------------------------------------------------------------------- +@pytest.mark.xdist_group(name="grpc_server") class TestTLSExecution: """Test remote execution over a TLS-encrypted channel. @@ -729,7 +732,7 @@ def tls_env_with_server(self, tmp_path_factory): pytest.skip("cuopt_grpc_server not found") port = int(os.environ.get("CUOPT_TEST_PORT_BASE", "18000")) + 850 - proc = subprocess.Popen( + proc = spawn_server( [ server_bin, "--port", @@ -742,12 +745,9 @@ def tls_env_with_server(self, tmp_path_factory): "--tls-key", os.path.join(cert_dir, "server.key"), ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, ) if not _wait_for_port(port, timeout=15): - proc.kill() - proc.wait() + kill_server(proc) pytest.fail("TLS cuopt_grpc_server failed to start within 15s") env = _tls_env(port, cert_dir, mtls=False) @@ -768,6 +768,7 @@ def test_lp_solve_tls(self, tls_env_with_server): # --------------------------------------------------------------------------- +@pytest.mark.xdist_group(name="grpc_server") class TestMTLSExecution: """Test remote execution over an mTLS-encrypted channel. @@ -787,7 +788,7 @@ def mtls_server_info(self, tmp_path_factory): pytest.skip("cuopt_grpc_server not found") port = int(os.environ.get("CUOPT_TEST_PORT_BASE", "18000")) + 900 - proc = subprocess.Popen( + proc = spawn_server( [ server_bin, "--port", @@ -803,22 +804,14 @@ def mtls_server_info(self, tmp_path_factory): os.path.join(cert_dir, "ca.crt"), "--require-client-cert", ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, ) if not _wait_for_port(port, timeout=15): - proc.kill() - proc.wait() + kill_server(proc) pytest.fail("mTLS cuopt_grpc_server failed to start within 15s") yield {"port": port, "cert_dir": cert_dir} - proc.send_signal(signal.SIGTERM) - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() + kill_server(proc) def test_lp_solve_mtls(self, mtls_server_info): """LP solve succeeds over an mTLS channel with valid client cert.""" diff --git a/python/cuopt/cuopt/tests/linear_programming/test_grpc_client.py b/python/cuopt/cuopt/tests/linear_programming/test_grpc_client.py index 31a039503f..37396927ba 100644 --- a/python/cuopt/cuopt/tests/linear_programming/test_grpc_client.py +++ b/python/cuopt/cuopt/tests/linear_programming/test_grpc_client.py @@ -63,6 +63,7 @@ def _infeasible_lp_problem(): return problem +@pytest.mark.xdist_group(name="grpc_server") @pytest.mark.filterwarnings("ignore::DeprecationWarning") class TestGrpcClient: grpc_port_offset = GRPC_PORT_OFFSET_CLIENT From 9b96fb7e557b3d06a343003bfe265c22849f86d1 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 7 Jul 2026 11:53:33 -0500 Subject: [PATCH 5/6] ci(test): trim verbose comments in grpc server/test wiring --- ci/run_cuopt_pytests.sh | 7 ++---- .../tests/fixtures/grpc_server_fixtures.py | 22 +++++-------------- .../test_cpu_only_execution.py | 6 ++--- 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/ci/run_cuopt_pytests.sh b/ci/run_cuopt_pytests.sh index 34923197d5..c3b047c888 100755 --- a/ci/run_cuopt_pytests.sh +++ b/ci/run_cuopt_pytests.sh @@ -36,11 +36,8 @@ rc=0 if [ "${IS_NIGHTLY}" = "nightly" ]; then pytest -s --cache-clear --reruns 2 --reruns-delay 5 -p cuopt_rerun_xml "$@" tests || rc=$? else - # --dist loadgroup keeps @pytest.mark.xdist_group tests on one worker, so - # all cuopt_grpc_server-backed classes share a single worker instead of - # duplicating a GPU server per worker (without loadgroup the markers are - # silently ignored). --max-worker-restart=0 disables respawning a crashed - # worker, so a crash fails those tests once rather than cascading. + # loadgroup keeps xdist_group (grpc server) tests on one worker; + # max-worker-restart=0 stops a crashed worker from respawning. pytest -s --cache-clear -n 4 --dist loadgroup --max-worker-restart=0 "$@" tests || rc=$? fi diff --git a/python/cuopt/cuopt/tests/fixtures/grpc_server_fixtures.py b/python/cuopt/cuopt/tests/fixtures/grpc_server_fixtures.py index 33b1b1776e..5a5f612101 100644 --- a/python/cuopt/cuopt/tests/fixtures/grpc_server_fixtures.py +++ b/python/cuopt/cuopt/tests/fixtures/grpc_server_fixtures.py @@ -117,10 +117,7 @@ def server_env(): def _set_pdeathsig(): - """Ask the kernel to SIGKILL this child if the spawning (pytest worker) - process dies. Ensures a crashed/killed worker can't orphan a GPU-holding - server. Linux-only; a no-op (best effort) elsewhere. - """ + """SIGKILL this child if the spawning worker dies (Linux; best effort).""" try: import ctypes @@ -133,16 +130,9 @@ def _set_pdeathsig(): def spawn_server(cmd, env=None): - """Start ``cuopt_grpc_server`` in its own process group with parent-death - cleanup. - - ``start_new_session=True`` makes the server a process-group leader, so its - ``--workers`` child shares the group and both can be reaped together (see - ``kill_server``). ``preexec_fn`` arms PR_SET_PDEATHSIG so an abnormally - exiting worker (e.g. GPU OOM abort) doesn't leave the server holding GPU - memory. Without this, servers leaked across tests/runs and stacked up until - the device OOMed. - """ + """Start ``cuopt_grpc_server`` in its own process group, dying with the + spawning worker, so it and its ``--workers`` child can be reaped together + (see ``kill_server``) and don't leak GPU memory.""" return subprocess.Popen( cmd, stdout=subprocess.DEVNULL, @@ -154,9 +144,7 @@ def spawn_server(cmd, env=None): def kill_server(proc): - """Terminate the server *and its whole process group* (parent + ``--workers`` - child). SIGTERM the group, wait, then SIGKILL any survivors. - """ + """Terminate the server's whole process group (parent + ``--workers`` child).""" if proc is None: return try: diff --git a/python/cuopt/cuopt/tests/linear_programming/test_cpu_only_execution.py b/python/cuopt/cuopt/tests/linear_programming/test_cpu_only_execution.py index 2b379eb3d1..3dcdfc5c60 100644 --- a/python/cuopt/cuopt/tests/linear_programming/test_cpu_only_execution.py +++ b/python/cuopt/cuopt/tests/linear_programming/test_cpu_only_execution.py @@ -27,10 +27,8 @@ from cuopt import linear_programming from cuopt.linear_programming.solver.solver_parameters import CUOPT_TIME_LIMIT -# This module is also re-executed as a plain script by _run_in_subprocess -# (python test_cpu_only_execution.py _impl_...), where conftest has not put -# the tests fixtures/ dir on sys.path. Add it so the import works in both the -# pytest and standalone-subprocess contexts. +# Also re-executed as a standalone script (_run_in_subprocess), where conftest +# hasn't added fixtures/ to sys.path; add it so this import works either way. sys.path.insert( 0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "fixtures") ) From c491ac4d9ab2944238547fc9df8f346d451e98c3 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Tue, 7 Jul 2026 11:54:47 -0500 Subject: [PATCH 6/6] style: apply pre-commit (ruff-format, pydocstyle) --- python/cuopt/cuopt/tests/fixtures/grpc_server_fixtures.py | 3 ++- .../cuopt/tests/linear_programming/test_cpu_only_execution.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/cuopt/cuopt/tests/fixtures/grpc_server_fixtures.py b/python/cuopt/cuopt/tests/fixtures/grpc_server_fixtures.py index 5a5f612101..39c6eba644 100644 --- a/python/cuopt/cuopt/tests/fixtures/grpc_server_fixtures.py +++ b/python/cuopt/cuopt/tests/fixtures/grpc_server_fixtures.py @@ -132,7 +132,8 @@ def _set_pdeathsig(): def spawn_server(cmd, env=None): """Start ``cuopt_grpc_server`` in its own process group, dying with the spawning worker, so it and its ``--workers`` child can be reaped together - (see ``kill_server``) and don't leak GPU memory.""" + (see ``kill_server``) and don't leak GPU memory. + """ return subprocess.Popen( cmd, stdout=subprocess.DEVNULL, diff --git a/python/cuopt/cuopt/tests/linear_programming/test_cpu_only_execution.py b/python/cuopt/cuopt/tests/linear_programming/test_cpu_only_execution.py index 3dcdfc5c60..8d0d335f24 100644 --- a/python/cuopt/cuopt/tests/linear_programming/test_cpu_only_execution.py +++ b/python/cuopt/cuopt/tests/linear_programming/test_cpu_only_execution.py @@ -30,7 +30,8 @@ # Also re-executed as a standalone script (_run_in_subprocess), where conftest # hasn't added fixtures/ to sys.path; add it so this import works either way. sys.path.insert( - 0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "fixtures") + 0, + os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "fixtures"), ) from grpc_server_fixtures import kill_server, spawn_server # noqa: E402