diff --git a/doc/html/power_save.shtml b/doc/html/power_save.shtml index b7de9d90861..c8be5cae4cc 100644 --- a/doc/html/power_save.shtml +++ b/doc/html/power_save.shtml @@ -304,6 +304,17 @@ SLURM_RESUME_FILE=/proc/1647372/fd/7: State=CLOUD nodes, the default is 90.

+
periodic_check_interval=# +
+ +
+

How often slurmctld runs periodic background checks, including job + time-limit handling, reservation checks, and node timer checks. Lower + values can reduce the delay before jobs progress after nodes resume + and register. Default is 30 seconds.

+
+
power_save_interval=#
diff --git a/doc/man/man5/slurm.conf.5 b/doc/man/man5/slurm.conf.5 index 3bdc37fa086..f0483dda00f 100644 --- a/doc/man/man5/slurm.conf.5 +++ b/doc/man/man5/slurm.conf.5 @@ -5488,6 +5488,14 @@ running before taking control. If this option is set this will not happen without the \fB-i\fR option. .IP +.TP +\fBperiodic_check_interval\fR=\# +How often slurmctld runs periodic background checks, including job time-limit +handling, reservation checks, and node timer checks. Lower values can reduce +the delay before jobs progress after nodes resume and register. Default is 30 +seconds. +.IP + .TP \fBpower_save_interval\fR How often the power_save thread looks to resume and suspend nodes. The diff --git a/src/slurmctld/controller.c b/src/slurmctld/controller.c index 5d502f2dc59..4d199ee3323 100644 --- a/src/slurmctld/controller.c +++ b/src/slurmctld/controller.c @@ -72,6 +72,7 @@ #include "src/common/log.h" #include "src/common/macros.h" #include "src/common/pack.h" +#include "src/common/parse_value.h" #include "src/common/port_mgr.h" #include "src/common/probes.h" #include "src/common/proc_args.h" @@ -621,6 +622,36 @@ static void _close_acct_storage_conn(void) slurm_persist_conn_recv_server_fini(); } +extern uint16_t get_periodic_check_interval(void) +{ + static time_t config_update = (time_t) -1; + static uint16_t periodic_check_interval = PERIODIC_TIMEOUT; + char *tmp_ptr; + uint16_t tmp_interval = PERIODIC_TIMEOUT; + + if (config_update == slurm_conf.last_update) + return periodic_check_interval; + + if ((tmp_ptr = conf_get_opt_str(slurm_conf.slurmctld_params, + "periodic_check_interval="))) { + if (s_p_handle_uint16(&tmp_interval, + "periodic_check_interval", + tmp_ptr) || + !tmp_interval || (tmp_interval == INFINITE16)) { + error("SlurmctldParameters option periodic_check_interval=%s " + "is invalid, using default %u", + tmp_ptr, PERIODIC_TIMEOUT); + tmp_interval = PERIODIC_TIMEOUT; + } + xfree(tmp_ptr); + } + + periodic_check_interval = tmp_interval; + config_update = slurm_conf.last_update; + + return periodic_check_interval; +} + /* main - slurmctld main function, start various threads and process RPCs */ int main(int argc, char **argv) { @@ -2740,7 +2771,8 @@ static void *_slurmctld_background(void *no_data) validate_all_reservations(true, true); - if (difftime(now, last_timelimit_time) >= PERIODIC_TIMEOUT) { + if (difftime(now, last_timelimit_time) >= + get_periodic_check_interval()) { lock_slurmctld(job_write_lock); now = time(NULL); last_timelimit_time = now; diff --git a/src/slurmctld/job_mgr.c b/src/slurmctld/job_mgr.c index f4b76b9f904..578b9f8ad48 100644 --- a/src/slurmctld/job_mgr.c +++ b/src/slurmctld/job_mgr.c @@ -9577,7 +9577,8 @@ void job_time_limit(void) } /* Give srun command warning message about pending timeout */ - if (job_ptr->end_time <= (now + PERIODIC_TIMEOUT * 2)) + if (job_ptr->end_time <= + (now + get_periodic_check_interval() * 2)) srun_timeout (job_ptr); /* @@ -19355,7 +19356,8 @@ extern void send_job_warn_signal(job_record_t *job_ptr, bool ignore_time) !(job_ptr->warn_flags & WARN_SENT) && (ignore_time || (job_ptr->warn_time && - ((job_ptr->warn_time + PERIODIC_TIMEOUT + time(NULL)) >= + ((job_ptr->warn_time + get_periodic_check_interval() + + time(NULL)) >= job_ptr->end_time)))) { /* * If --signal B option was not specified, diff --git a/src/slurmctld/slurmctld.h b/src/slurmctld/slurmctld.h index a6c56214d02..404d47e63aa 100644 --- a/src/slurmctld/slurmctld.h +++ b/src/slurmctld/slurmctld.h @@ -2122,6 +2122,7 @@ extern double calc_job_billable_tres(job_record_t *job_ptr, time_t start_time, * resume_after - Resume a down|drain node after resume_after time. */ extern void check_node_timers(void); +extern uint16_t get_periodic_check_interval(void); /* * Send warning signal to job before end time. diff --git a/testsuite/python/tests/test_141_1.py b/testsuite/python/tests/test_141_1.py index a6218a579cc..64094811291 100644 --- a/testsuite/python/tests/test_141_1.py +++ b/testsuite/python/tests/test_141_1.py @@ -15,6 +15,7 @@ suspend_time = 10 suspend_timeout = 10 resume_timeout = 10 +periodic_check_interval = 2 @pytest.fixture(scope="module", autouse=True) @@ -40,6 +41,9 @@ def setup(): # Mark nodes as IDLE, regardless of current state, when suspending nodes with # SuspendProgram so that nodes will be eligible to be resumed at a later time atf.require_config_parameter_includes("SlurmctldParameters", "idle_on_node_suspend") + atf.require_config_parameter_includes( + "SlurmctldParameters", f"periodic_check_interval={periodic_check_interval}" + ) # Register the cloud node in slurm.conf atf.require_config_parameter( @@ -109,6 +113,40 @@ def kill_slurmctld(): # Tests +def test_periodic_check_interval(): + """Test periodic_check_interval advances a CONFIGURING job after node registration.""" + job_id = atf.submit_job_sbatch("-p cloud1 --wrap 'srun sleep 10'", fatal=True) + atf.wait_for_node_state(f"{node_prefix}1", "ALLOCATED", timeout=5, fatal=True) + atf.wait_for_node_state(f"{node_prefix}1", "POWERING_UP", fatal=True) + assert "CONFIGURING" == atf.get_job_parameter( + job_id, "JobState", default="NOT_FOUND", quiet=True + ), "Submitted job should be in CONFIGURING state while its ALLOCATED cloud node is POWERING_UP" + + # TODO: Wait 2 seconds to avoid race condition between slurmd and slurmctld + # Remove once bug 16459 is fixed. + time.sleep(2) + + atf.run_command( + f"{atf.properties['slurm-sbin-dir']}/slurmd -b -N {node_prefix}1 --conf 'feature=f1'", + fatal=True, + user="root", + ) + + atf.wait_for_node_state( + f"{node_prefix}1", + "POWERING_UP", + reverse=True, + timeout=resume_timeout + 5, + fatal=True, + ) + + assert atf.wait_for_job_state( + job_id, + "RUNNING", + timeout=periodic_check_interval + 5, + ) + + # Test state cycle of cloud nodes: POWERED_DOWN, POWERING_UP, IDLE, # POWERING_DOWN, POWERED_DOWN def test_cloud_state_cycle():