Add cause-aware batch job requeue limits

CHANGELOG/slurm-26.11.md

@@ -0,0 +1,3 @@
+## Changes in Slurm (unreleased)
+
+* slurm.conf - Add MaxNodeFailRequeue and MaxPreemptRequeue to bound node-failure and preemption requeues independently of MaxBatchRequeue.

contribs/perlapi/libslurm/perl/conf.c

@@ -155,6 +155,8 @@ int slurm_ctl_conf_to_hv(slurm_conf_t *conf, HV *hv)
 
 	STORE_FIELD(hv, conf, max_array_sz, uint32_t);
 	STORE_FIELD(hv, conf, max_batch_requeue, uint32_t);
+	STORE_FIELD(hv, conf, max_node_fail_requeue, uint32_t);
+	STORE_FIELD(hv, conf, max_preempt_requeue, uint32_t);
 	STORE_FIELD(hv, conf, max_dbd_msgs, uint32_t);
 
 	STORE_FIELD(hv, conf, max_job_cnt, uint32_t);
@@ -451,6 +453,8 @@ int hv_to_slurm_ctl_conf(HV *hv, slurm_conf_t *conf)
 	FETCH_FIELD(hv, conf, mail_prog, charp, FALSE);
 	FETCH_FIELD(hv, conf, max_array_sz, uint32_t, TRUE);
 	FETCH_FIELD(hv, conf, max_batch_requeue, uint32_t, TRUE);
+	FETCH_FIELD(hv, conf, max_node_fail_requeue, uint32_t, TRUE);
+	FETCH_FIELD(hv, conf, max_preempt_requeue, uint32_t, TRUE);
 	FETCH_FIELD(hv, conf, max_dbd_msgs, uint32_t, TRUE);
 	FETCH_FIELD(hv, conf, max_job_cnt, uint32_t, TRUE);
 	FETCH_FIELD(hv, conf, max_job_id, uint32_t, FALSE);

doc/man/man5/slurm.conf.5

@@ -2402,12 +2402,28 @@ The value may not exceed 65533.
 
 .TP
 \fBMaxBatchRequeue\fR
-Maximum number of times a batch job may be automatically requeued before
-being marked as JobHeldAdmin. (Mainly useful when the \fBSchedulerParameters\fR
-option \fBnohold_on_prolog_fail\fR is enabled.)
+Maximum number of times a batch job may be requeued due to launch or
+prolog failures before being held. (Mainly useful when the
+\fBSchedulerParameters\fR option \fBnohold_on_prolog_fail\fR is enabled.)
 The default value is 5.
 .IP
 
+.TP
+\fBMaxNodeFailRequeue\fR
+Maximum number of times a batch job may be requeued after an allocated
+node fails before the job is held. This can limit jobs suspected of repeatedly
+triggering node failures from being requeued indefinitely; Slurm does not
+diagnose whether the job or node caused the failure. A value of 0 disables the
+limit. The default value is 5.
+.IP
+
+.TP
+\fBMaxPreemptRequeue\fR
+Maximum number of times a batch job may be requeued due to preemption or
+operator requeue before the job is held. A value of 0 disables the limit.
+The default value is 0.
+.IP
+
 .TP
 \fBNodeFeaturesPlugins\fR
 Identifies the plugins to be used for support of node features which can

slurm/slurm.h

@@ -658,6 +658,8 @@ enum job_state_reason {
 					    * (Unknown) */
 	WAIT_MAX_POWERED_NODES,            /* max_powered_nodes reached */
 	WAIT_MPI_PORTS_BUSY,		   /* MPI resv_ports busy */
+	WAIT_MAX_NODE_FAIL_REQUEUE,	   /* MaxNodeFailRequeue reached */
+	WAIT_MAX_PREEMPT_REQUEUE,	   /* MaxPreemptRequeue reached */
 	REASON_END, /* end of table */
 };
 
@@ -3145,7 +3147,12 @@ typedef struct {
 	char *mail_domain;	/* default domain to append to usernames */
 	char *mail_prog;	/* pathname of mail program */
 	uint32_t max_array_sz;	/* Maximum job array size */
-	uint32_t max_batch_requeue; /* maximum number of requeues */
+	uint32_t max_batch_requeue; /* maximum number of launch-failure
+				     * requeues */
+	uint32_t max_node_fail_requeue; /* maximum number of node-failure
+					 * requeues */
+	uint32_t max_preempt_requeue; /* maximum number of preemption/operator
+				       * requeues */
 	uint32_t max_dbd_msgs;	/* maximum number of messages queued while DBD
 				 * is not connected */
 	uint32_t max_job_cnt;	/* maximum number of active jobs */

src/api/config_info.c

@@ -853,6 +853,12 @@ extern void *slurm_ctl_conf_2_key_pairs(slurm_conf_t *conf)
 	add_key_pair(ret_list, "MaxBatchRequeue", "%u",
 		     conf->max_batch_requeue);
 
+	add_key_pair(ret_list, "MaxNodeFailRequeue", "%u",
+		     conf->max_node_fail_requeue);
+
+	add_key_pair(ret_list, "MaxPreemptRequeue", "%u",
+		     conf->max_preempt_requeue);
+
 	add_key_pair(ret_list, "MaxDBDMsgs", "%u", conf->max_dbd_msgs);
 
 	add_key_pair(ret_list, "MaxJobCount", "%u", conf->max_job_cnt);

src/common/job_record.c

@@ -2198,6 +2198,11 @@ extern void job_record_pack_common(job_record_t *dump_job_ptr,
 		pack32(dump_job_ptr->req_switch, buffer);
 		pack_time(dump_job_ptr->resize_time, buffer);
 		pack16(dump_job_ptr->restart_cnt, buffer);
+		if (for_state &&
+		    (protocol_version >= SLURM_26_11_PROTOCOL_VERSION)) {
+			pack32(dump_job_ptr->node_fail_requeue_cnt, buffer);
+			pack32(dump_job_ptr->preempt_requeue_cnt, buffer);
+		}
 		packstr(dump_job_ptr->resv_name, buffer);
 		packstr(dump_job_ptr->resv_ports, buffer);
 
@@ -2483,6 +2488,10 @@ extern int job_record_unpack_common(job_record_t *job_ptr,
 		safe_unpack32(&job_ptr->req_switch, buffer);
 		safe_unpack_time(&job_ptr->resize_time, buffer);
 		safe_unpack16(&job_ptr->restart_cnt, buffer);
+		if (protocol_version >= SLURM_26_11_PROTOCOL_VERSION) {
+			safe_unpack32(&job_ptr->node_fail_requeue_cnt, buffer);
+			safe_unpack32(&job_ptr->preempt_requeue_cnt, buffer);
+		}
 		safe_unpackstr(&job_ptr->resv_name, buffer);
 		safe_unpackstr(&job_ptr->resv_ports, buffer);
 

src/common/job_record.h

@@ -471,6 +471,8 @@ struct job_record {
 	slurmdb_qos_rec_t *qos_blocking_ptr; /* internal use only, DON'T PACK */
 	uint8_t reboot;			/* node reboot requested before start */
 	uint16_t restart_cnt;		/* count of restarts */
+	uint32_t node_fail_requeue_cnt;	/* count of node-failure requeues */
+	uint32_t preempt_requeue_cnt;	/* count of preemption/operator requeues */
 	time_t resize_time;		/* time of latest size change */
 	uint32_t resv_id;		/* reservation ID */
 	list_t *resv_list;		/* Filled in if the job is requesting

src/common/job_state_reason.c

@@ -306,6 +306,7 @@ const static entry_t jsra[] = {
 		.str = "AssocMaxSubmitJobLimit",
 	},
 	[WAIT_MAX_REQUEUE] = {
+		/* Keep legacy spelling for backward compatibility. */
 		.str = "JobHoldMaxRequeue",
 	},
 	[WAIT_ARRAY_TASK_LIMIT] = {
@@ -882,6 +883,12 @@ const static entry_t jsra[] = {
 	[WAIT_MPI_PORTS_BUSY] = {
 		.str = "MpiPortsBusy",
 	},
+	[WAIT_MAX_NODE_FAIL_REQUEUE] = {
+		.str = "JobHeldMaxNodeFailRequeue",
+	},
+	[WAIT_MAX_PREEMPT_REQUEUE] = {
+		.str = "JobHeldMaxPreemptRequeue",
+	},
 };
 
 extern const char *job_state_reason_string(enum job_state_reason inx)

src/common/read_config.c

@@ -322,6 +322,8 @@ s_p_options_t slurm_conf_options[] = {
 	{"MailProg", S_P_STRING},
 	{"MaxArraySize", S_P_UINT32},
 	{"MaxBatchRequeue", S_P_UINT32},
+	{"MaxNodeFailRequeue", S_P_UINT32},
+	{"MaxPreemptRequeue", S_P_UINT32},
 	{"MaxDBDMsgs", S_P_UINT32},
 	{"MaxJobCount", S_P_UINT32},
 	{"MaxJobId", S_P_UINT32},
@@ -2970,6 +2972,8 @@ extern void init_slurm_conf(slurm_conf_t *conf)
 	xfree(conf->mail_prog);
 	conf->max_array_sz = NO_VAL;
 	conf->max_batch_requeue = NO_VAL;
+	conf->max_node_fail_requeue = NO_VAL;
+	conf->max_preempt_requeue = NO_VAL;
 	conf->max_dbd_msgs = 0;
 	conf->max_job_cnt = NO_VAL;
 	conf->max_job_id = NO_VAL;
@@ -4397,6 +4401,14 @@ static int _validate_and_set_defaults(slurm_conf_t *conf,
 			    hashtbl))
 		conf->max_batch_requeue = DEFAULT_MAX_BATCH_REQUEUE;
 
+	if (!s_p_get_uint32(&conf->max_node_fail_requeue, "MaxNodeFailRequeue",
+			    hashtbl))
+		conf->max_node_fail_requeue = DEFAULT_MAX_NODE_FAIL_REQUEUE;
+
+	if (!s_p_get_uint32(&conf->max_preempt_requeue, "MaxPreemptRequeue",
+			    hashtbl))
+		conf->max_preempt_requeue = DEFAULT_MAX_PREEMPT_REQUEUE;
+
 	if (!s_p_get_uint32(&conf->max_dbd_msgs, "MaxDBDMsgs", hashtbl))
 		conf->max_dbd_msgs = 0;
 	else if (conf->max_dbd_msgs < DEFAULT_MAX_DBD_MSGS) {

src/common/read_config.h

@@ -119,6 +119,8 @@ typedef struct node_record node_record_t;
 #define DEFAULT_MAIL_PROG_ALT       "/usr/bin/mail"
 #define DEFAULT_MAX_ARRAY_SIZE      1001
 #define DEFAULT_MAX_BATCH_REQUEUE   5
+#define DEFAULT_MAX_NODE_FAIL_REQUEUE 5
+#define DEFAULT_MAX_PREEMPT_REQUEUE 0
 #define DEFAULT_MAX_DBD_MSGS        10000
 #define DEFAULT_MAX_JOB_COUNT       10000
 #define DEFAULT_MAX_JOB_ID          0x03ff0000

src/common/slurm_protocol_pack.c

@@ -3971,6 +3971,10 @@ static void _pack_slurm_conf(const slurm_conf_t *conf,
 
 		pack32(conf->max_array_sz, buffer);
 		pack32(conf->max_batch_requeue, buffer);
+		if (protocol_version >= SLURM_26_11_PROTOCOL_VERSION) {
+			pack32(conf->max_node_fail_requeue, buffer);
+			pack32(conf->max_preempt_requeue, buffer);
+		}
 		pack32(conf->max_dbd_msgs, buffer);
 		packstr(conf->mail_domain, buffer);
 		packstr(conf->mail_prog, buffer);
@@ -4789,6 +4793,10 @@ static int _unpack_slurm_conf(slurm_conf_t **conf_ptr,
 
 		safe_unpack32(&conf->max_array_sz, buffer);
 		safe_unpack32(&conf->max_batch_requeue, buffer);
+		if (protocol_version >= SLURM_26_11_PROTOCOL_VERSION) {
+			safe_unpack32(&conf->max_node_fail_requeue, buffer);
+			safe_unpack32(&conf->max_preempt_requeue, buffer);
+		}
 		safe_unpack32(&conf->max_dbd_msgs, buffer);
 		safe_unpackstr(&conf->mail_domain, buffer);
 		safe_unpackstr(&conf->mail_prog, buffer);
@@ -5521,6 +5529,11 @@ static int _unpack_slurm_conf(slurm_conf_t **conf_ptr,
 		safe_unpackstr(&conf->x11_params, buffer);
 	}
 
+	if (protocol_version < SLURM_26_11_PROTOCOL_VERSION) {
+		conf->max_node_fail_requeue = DEFAULT_MAX_NODE_FAIL_REQUEUE;
+		conf->max_preempt_requeue = DEFAULT_MAX_PREEMPT_REQUEUE;
+	}
+
 	*conf_ptr = conf;
 	return SLURM_SUCCESS;
 

src/plugins/data_parser/v0.0.46/parsers.c

@@ -12442,6 +12442,8 @@ static const parser_t PARSER_ARRAY(SLURM_CONF)[] = {
 	add_parse(STRING, mail_prog, "MailProg", "Pathname of mail program"),
 	add_parse(UINT32, max_array_sz, "MaxArraySize", "Maximum job array size"),
 	add_parse(UINT32, max_batch_requeue, "MaxBatchRequeue", "Max times a batch job may be auto-requeued before being held"),
+	add_parse(UINT32, max_node_fail_requeue, "MaxNodeFailRequeue", "Max node-failure requeues before hold (0=disabled)"),
+	add_parse(UINT32, max_preempt_requeue, "MaxPreemptRequeue", "Max preemption/operator requeues before hold (0=disabled)"),
 	add_parse(UINT32, max_dbd_msgs, "MaxDBDMsgs", "Maximum number of messages queued while DBD is not connected"),
 	add_parse(UINT32, max_job_cnt, "MaxJobCount", "Maximum number of active jobs"),
 	add_parse(UINT32, max_job_id, "MaxJobId", "Maximum job id before wrapping back to FirstJobId"),
@@ -12710,6 +12712,8 @@ static const parser_t PARSER_ARRAY(SLURM_CONF_META)[] = {
 	add_skip(mail_prog),
 	add_skip(max_array_sz),
 	add_skip(max_batch_requeue),
+	add_skip(max_node_fail_requeue),
+	add_skip(max_preempt_requeue),
 	add_skip(max_dbd_msgs),
 	add_skip(max_job_cnt),
 	add_skip(max_job_id),