Make slurmctld periodic check interval configurable

doc/html/power_save.shtml

@@ -304,6 +304,17 @@ SLURM_RESUME_FILE=/proc/1647372/fd/7:
         State=CLOUD nodes, the default is 90.</p>
       </dd>
 
+      <dt id="periodic_check_interval"><b>periodic_check_interval=#</b><a class=
+      "slurm_link" href="#periodic_check_interval"></a>
+      </dt>
+
+      <dd>
+        <p>How often slurmctld runs periodic background checks, including job
+        time-limit handling, reservation checks, and node timer checks. Lower
+        values can reduce the delay before jobs progress after nodes resume
+        and register. Default is 30 seconds.</p>
+      </dd>
+
       <dt id="power_save_interval"><b>power_save_interval=#</b><a class=
       "slurm_link" href="#power_save_interval"></a>
       </dt>

doc/man/man5/slurm.conf.5

@@ -5488,6 +5488,14 @@ running before taking control. If this option is set this will not happen
 without the \fB-i\fR option.
 .IP
 
+.TP
+\fBperiodic_check_interval\fR=\#
+How often slurmctld runs periodic background checks, including job time-limit
+handling, reservation checks, and node timer checks. Lower values can reduce
+the delay before jobs progress after nodes resume and register. Default is 30
+seconds.
+.IP
+
 .TP
 \fBpower_save_interval\fR
 How often the power_save thread looks to resume and suspend nodes. The

src/slurmctld/controller.c

@@ -72,6 +72,7 @@
 #include "src/common/log.h"
 #include "src/common/macros.h"
 #include "src/common/pack.h"
+#include "src/common/parse_value.h"
 #include "src/common/port_mgr.h"
 #include "src/common/probes.h"
 #include "src/common/proc_args.h"
@@ -621,6 +622,36 @@ static void _close_acct_storage_conn(void)
 	slurm_persist_conn_recv_server_fini();
 }
 
+extern uint16_t get_periodic_check_interval(void)
+{
+	static time_t config_update = (time_t) -1;
+	static uint16_t periodic_check_interval = PERIODIC_TIMEOUT;
+	char *tmp_ptr;
+	uint16_t tmp_interval = PERIODIC_TIMEOUT;
+
+	if (config_update == slurm_conf.last_update)
+		return periodic_check_interval;
+
+	if ((tmp_ptr = conf_get_opt_str(slurm_conf.slurmctld_params,
+					"periodic_check_interval="))) {
+		if (s_p_handle_uint16(&tmp_interval,
+				      "periodic_check_interval",
+				      tmp_ptr) ||
+		    !tmp_interval || (tmp_interval == INFINITE16)) {
+			error("SlurmctldParameters option periodic_check_interval=%s "
+			      "is invalid, using default %u",
+			      tmp_ptr, PERIODIC_TIMEOUT);
+			tmp_interval = PERIODIC_TIMEOUT;
+		}
+		xfree(tmp_ptr);
+	}
+
+	periodic_check_interval = tmp_interval;
+	config_update = slurm_conf.last_update;
+
+	return periodic_check_interval;
+}
+
 /* main - slurmctld main function, start various threads and process RPCs */
 int main(int argc, char **argv)
 {
@@ -2740,7 +2771,8 @@ static void *_slurmctld_background(void *no_data)
 
 		validate_all_reservations(true, true);
 
-		if (difftime(now, last_timelimit_time) >= PERIODIC_TIMEOUT) {
+		if (difftime(now, last_timelimit_time) >=
+		    get_periodic_check_interval()) {
 			lock_slurmctld(job_write_lock);
 			now = time(NULL);
 			last_timelimit_time = now;

src/slurmctld/job_mgr.c

@@ -9577,7 +9577,8 @@ void job_time_limit(void)
 		}
 
 		/* Give srun command warning message about pending timeout */
-		if (job_ptr->end_time <= (now + PERIODIC_TIMEOUT * 2))
+		if (job_ptr->end_time <=
+		    (now + get_periodic_check_interval() * 2))
 			srun_timeout (job_ptr);
 
 		/*
@@ -19355,7 +19356,8 @@ extern void send_job_warn_signal(job_record_t *job_ptr, bool ignore_time)
 	    !(job_ptr->warn_flags & WARN_SENT) &&
 	    (ignore_time ||
 	     (job_ptr->warn_time &&
-	      ((job_ptr->warn_time + PERIODIC_TIMEOUT + time(NULL)) >=
+	      ((job_ptr->warn_time + get_periodic_check_interval() +
+		time(NULL)) >=
 	       job_ptr->end_time)))) {
 		/*
 		 * If --signal B option was not specified,

src/slurmctld/slurmctld.h

@@ -2122,6 +2122,7 @@ extern double calc_job_billable_tres(job_record_t *job_ptr, time_t start_time,
  * resume_after - Resume a down|drain node after resume_after time.
  */
 extern void check_node_timers(void);
+extern uint16_t get_periodic_check_interval(void);
 
 /*
  * Send warning signal to job before end time.

testsuite/python/tests/test_141_1.py

@@ -15,6 +15,7 @@
 suspend_time = 10
 suspend_timeout = 10
 resume_timeout = 10
+periodic_check_interval = 2
 
 
 @pytest.fixture(scope="module", autouse=True)
@@ -40,6 +41,9 @@ def setup():
     # Mark nodes as IDLE, regardless of current state, when suspending nodes with
     # SuspendProgram so that nodes will be eligible to be resumed at a later time
     atf.require_config_parameter_includes("SlurmctldParameters", "idle_on_node_suspend")
+    atf.require_config_parameter_includes(
+        "SlurmctldParameters", f"periodic_check_interval={periodic_check_interval}"
+    )
 
     # Register the cloud node in slurm.conf
     atf.require_config_parameter(
@@ -109,6 +113,40 @@ def kill_slurmctld():
 
 
 # Tests
+def test_periodic_check_interval():
+    """Test periodic_check_interval advances a CONFIGURING job after node registration."""
+    job_id = atf.submit_job_sbatch("-p cloud1 --wrap 'srun sleep 10'", fatal=True)
+    atf.wait_for_node_state(f"{node_prefix}1", "ALLOCATED", timeout=5, fatal=True)
+    atf.wait_for_node_state(f"{node_prefix}1", "POWERING_UP", fatal=True)
+    assert "CONFIGURING" == atf.get_job_parameter(
+        job_id, "JobState", default="NOT_FOUND", quiet=True
+    ), "Submitted job should be in CONFIGURING state while its ALLOCATED cloud node is POWERING_UP"
+
+    # TODO: Wait 2 seconds to avoid race condition between slurmd and slurmctld
+    #       Remove once bug 16459 is fixed.
+    time.sleep(2)
+
+    atf.run_command(
+        f"{atf.properties['slurm-sbin-dir']}/slurmd -b -N {node_prefix}1 --conf 'feature=f1'",
+        fatal=True,
+        user="root",
+    )
+
+    atf.wait_for_node_state(
+        f"{node_prefix}1",
+        "POWERING_UP",
+        reverse=True,
+        timeout=resume_timeout + 5,
+        fatal=True,
+    )
+
+    assert atf.wait_for_job_state(
+        job_id,
+        "RUNNING",
+        timeout=periodic_check_interval + 5,
+    )
+
+
 # Test state cycle of cloud nodes: POWERED_DOWN, POWERING_UP, IDLE,
 # POWERING_DOWN, POWERED_DOWN
 def test_cloud_state_cycle():