diff --git a/doc/html/power_save.shtml b/doc/html/power_save.shtml
index b7de9d90861..c8be5cae4cc 100644
--- a/doc/html/power_save.shtml
+++ b/doc/html/power_save.shtml
@@ -304,6 +304,17 @@ SLURM_RESUME_FILE=/proc/1647372/fd/7:
State=CLOUD nodes, the default is 90.
+ periodic_check_interval=#
+
+
+
+ How often slurmctld runs periodic background checks, including job
+ time-limit handling, reservation checks, and node timer checks. Lower
+ values can reduce the delay before jobs progress after nodes resume
+ and register. Default is 30 seconds.
+
+
power_save_interval=#
diff --git a/doc/man/man5/slurm.conf.5 b/doc/man/man5/slurm.conf.5
index 3bdc37fa086..f0483dda00f 100644
--- a/doc/man/man5/slurm.conf.5
+++ b/doc/man/man5/slurm.conf.5
@@ -5488,6 +5488,14 @@ running before taking control. If this option is set this will not happen
without the \fB-i\fR option.
.IP
+.TP
+\fBperiodic_check_interval\fR=\#
+How often slurmctld runs periodic background checks, including job time-limit
+handling, reservation checks, and node timer checks. Lower values can reduce
+the delay before jobs progress after nodes resume and register. Default is 30
+seconds.
+.IP
+
.TP
\fBpower_save_interval\fR
How often the power_save thread looks to resume and suspend nodes. The
diff --git a/src/slurmctld/controller.c b/src/slurmctld/controller.c
index 5d502f2dc59..4d199ee3323 100644
--- a/src/slurmctld/controller.c
+++ b/src/slurmctld/controller.c
@@ -72,6 +72,7 @@
#include "src/common/log.h"
#include "src/common/macros.h"
#include "src/common/pack.h"
+#include "src/common/parse_value.h"
#include "src/common/port_mgr.h"
#include "src/common/probes.h"
#include "src/common/proc_args.h"
@@ -621,6 +622,36 @@ static void _close_acct_storage_conn(void)
slurm_persist_conn_recv_server_fini();
}
+extern uint16_t get_periodic_check_interval(void)
+{
+ static time_t config_update = (time_t) -1;
+ static uint16_t periodic_check_interval = PERIODIC_TIMEOUT;
+ char *tmp_ptr;
+ uint16_t tmp_interval = PERIODIC_TIMEOUT;
+
+ if (config_update == slurm_conf.last_update)
+ return periodic_check_interval;
+
+ if ((tmp_ptr = conf_get_opt_str(slurm_conf.slurmctld_params,
+ "periodic_check_interval="))) {
+ if (s_p_handle_uint16(&tmp_interval,
+ "periodic_check_interval",
+ tmp_ptr) ||
+ !tmp_interval || (tmp_interval == INFINITE16)) {
+ error("SlurmctldParameters option periodic_check_interval=%s "
+ "is invalid, using default %u",
+ tmp_ptr, PERIODIC_TIMEOUT);
+ tmp_interval = PERIODIC_TIMEOUT;
+ }
+ xfree(tmp_ptr);
+ }
+
+ periodic_check_interval = tmp_interval;
+ config_update = slurm_conf.last_update;
+
+ return periodic_check_interval;
+}
+
/* main - slurmctld main function, start various threads and process RPCs */
int main(int argc, char **argv)
{
@@ -2740,7 +2771,8 @@ static void *_slurmctld_background(void *no_data)
validate_all_reservations(true, true);
- if (difftime(now, last_timelimit_time) >= PERIODIC_TIMEOUT) {
+ if (difftime(now, last_timelimit_time) >=
+ get_periodic_check_interval()) {
lock_slurmctld(job_write_lock);
now = time(NULL);
last_timelimit_time = now;
diff --git a/src/slurmctld/job_mgr.c b/src/slurmctld/job_mgr.c
index f4b76b9f904..578b9f8ad48 100644
--- a/src/slurmctld/job_mgr.c
+++ b/src/slurmctld/job_mgr.c
@@ -9577,7 +9577,8 @@ void job_time_limit(void)
}
/* Give srun command warning message about pending timeout */
- if (job_ptr->end_time <= (now + PERIODIC_TIMEOUT * 2))
+ if (job_ptr->end_time <=
+ (now + get_periodic_check_interval() * 2))
srun_timeout (job_ptr);
/*
@@ -19355,7 +19356,8 @@ extern void send_job_warn_signal(job_record_t *job_ptr, bool ignore_time)
!(job_ptr->warn_flags & WARN_SENT) &&
(ignore_time ||
(job_ptr->warn_time &&
- ((job_ptr->warn_time + PERIODIC_TIMEOUT + time(NULL)) >=
+ ((job_ptr->warn_time + get_periodic_check_interval() +
+ time(NULL)) >=
job_ptr->end_time)))) {
/*
* If --signal B option was not specified,
diff --git a/src/slurmctld/slurmctld.h b/src/slurmctld/slurmctld.h
index a6c56214d02..404d47e63aa 100644
--- a/src/slurmctld/slurmctld.h
+++ b/src/slurmctld/slurmctld.h
@@ -2122,6 +2122,7 @@ extern double calc_job_billable_tres(job_record_t *job_ptr, time_t start_time,
* resume_after - Resume a down|drain node after resume_after time.
*/
extern void check_node_timers(void);
+extern uint16_t get_periodic_check_interval(void);
/*
* Send warning signal to job before end time.
diff --git a/testsuite/python/tests/test_141_1.py b/testsuite/python/tests/test_141_1.py
index a6218a579cc..64094811291 100644
--- a/testsuite/python/tests/test_141_1.py
+++ b/testsuite/python/tests/test_141_1.py
@@ -15,6 +15,7 @@
suspend_time = 10
suspend_timeout = 10
resume_timeout = 10
+periodic_check_interval = 2
@pytest.fixture(scope="module", autouse=True)
@@ -40,6 +41,9 @@ def setup():
# Mark nodes as IDLE, regardless of current state, when suspending nodes with
# SuspendProgram so that nodes will be eligible to be resumed at a later time
atf.require_config_parameter_includes("SlurmctldParameters", "idle_on_node_suspend")
+ atf.require_config_parameter_includes(
+ "SlurmctldParameters", f"periodic_check_interval={periodic_check_interval}"
+ )
# Register the cloud node in slurm.conf
atf.require_config_parameter(
@@ -109,6 +113,40 @@ def kill_slurmctld():
# Tests
+def test_periodic_check_interval():
+ """Test periodic_check_interval advances a CONFIGURING job after node registration."""
+ job_id = atf.submit_job_sbatch("-p cloud1 --wrap 'srun sleep 10'", fatal=True)
+ atf.wait_for_node_state(f"{node_prefix}1", "ALLOCATED", timeout=5, fatal=True)
+ atf.wait_for_node_state(f"{node_prefix}1", "POWERING_UP", fatal=True)
+ assert "CONFIGURING" == atf.get_job_parameter(
+ job_id, "JobState", default="NOT_FOUND", quiet=True
+ ), "Submitted job should be in CONFIGURING state while its ALLOCATED cloud node is POWERING_UP"
+
+ # TODO: Wait 2 seconds to avoid race condition between slurmd and slurmctld
+ # Remove once bug 16459 is fixed.
+ time.sleep(2)
+
+ atf.run_command(
+ f"{atf.properties['slurm-sbin-dir']}/slurmd -b -N {node_prefix}1 --conf 'feature=f1'",
+ fatal=True,
+ user="root",
+ )
+
+ atf.wait_for_node_state(
+ f"{node_prefix}1",
+ "POWERING_UP",
+ reverse=True,
+ timeout=resume_timeout + 5,
+ fatal=True,
+ )
+
+ assert atf.wait_for_job_state(
+ job_id,
+ "RUNNING",
+ timeout=periodic_check_interval + 5,
+ )
+
+
# Test state cycle of cloud nodes: POWERED_DOWN, POWERING_UP, IDLE,
# POWERING_DOWN, POWERED_DOWN
def test_cloud_state_cycle():