From d3d8e8bf40d572370a6d10d3a2aad03ea0811329 Mon Sep 17 00:00:00 2001 From: Yash Gupta Date: Mon, 29 Jun 2026 09:28:23 -0700 Subject: [PATCH] Add cause-aware batch job requeue limits Add MaxNodeFailRequeue and MaxPreemptRequeue to bound node-failure and preemption/operator requeues independently from MaxBatchRequeue. Each cause keeps its own per-job counter. Jobs that exceed the configured limit are held with JobHeldMaxNodeFailRequeue or JobHeldMaxPreemptRequeue, while a limit of 0 disables the hold. scontrol release resets the counters so a released job gets a fresh allowance. The new config fields and job counters are packed behind the 26.11 protocol/state version guard. The change also updates config reporting, data parser coverage, Perl config bindings, reason-code docs, slurm.conf(5), the changelog, and ATF tests for the new limits and exemption cases. Changelog: Add MaxNodeFailRequeue and MaxPreemptRequeue to bound requeues by cause. Signed-off-by: Yash Gupta --- CHANGELOG/slurm-26.11.md | 3 + contribs/perlapi/libslurm/perl/conf.c | 4 + doc/man/man5/slurm.conf.5 | 22 +- slurm/slurm.h | 9 +- src/api/config_info.c | 6 + src/common/job_record.c | 9 + src/common/job_record.h | 2 + src/common/job_state_reason.c | 7 + src/common/read_config.c | 12 + src/common/read_config.h | 2 + src/common/slurm_protocol_pack.c | 13 + src/plugins/data_parser/v0.0.46/parsers.c | 4 + src/slurmctld/job_mgr.c | 102 +++- src/slurmctld/job_scheduler.c | 2 + src/slurmctld/node_scheduler.c | 4 +- src/slurmctld/proc_req.c | 2 + src/slurmd/common/slurmstepd_init.c | 4 + testsuite/python/tests/test_105_11.py | 659 ++++++++++++++++++++++ 18 files changed, 854 insertions(+), 12 deletions(-) create mode 100644 CHANGELOG/slurm-26.11.md create mode 100644 testsuite/python/tests/test_105_11.py diff --git a/CHANGELOG/slurm-26.11.md b/CHANGELOG/slurm-26.11.md new file mode 100644 index 00000000000..c1d64f9f6a0 --- /dev/null +++ b/CHANGELOG/slurm-26.11.md @@ -0,0 +1,3 @@ +## Changes in Slurm (unreleased) + +* slurm.conf - Add MaxNodeFailRequeue and MaxPreemptRequeue to bound node-failure and preemption requeues independently of MaxBatchRequeue. diff --git a/contribs/perlapi/libslurm/perl/conf.c b/contribs/perlapi/libslurm/perl/conf.c index b0e3e2e41c1..50c87fcd346 100644 --- a/contribs/perlapi/libslurm/perl/conf.c +++ b/contribs/perlapi/libslurm/perl/conf.c @@ -155,6 +155,8 @@ int slurm_ctl_conf_to_hv(slurm_conf_t *conf, HV *hv) STORE_FIELD(hv, conf, max_array_sz, uint32_t); STORE_FIELD(hv, conf, max_batch_requeue, uint32_t); + STORE_FIELD(hv, conf, max_node_fail_requeue, uint32_t); + STORE_FIELD(hv, conf, max_preempt_requeue, uint32_t); STORE_FIELD(hv, conf, max_dbd_msgs, uint32_t); STORE_FIELD(hv, conf, max_job_cnt, uint32_t); @@ -451,6 +453,8 @@ int hv_to_slurm_ctl_conf(HV *hv, slurm_conf_t *conf) FETCH_FIELD(hv, conf, mail_prog, charp, FALSE); FETCH_FIELD(hv, conf, max_array_sz, uint32_t, TRUE); FETCH_FIELD(hv, conf, max_batch_requeue, uint32_t, TRUE); + FETCH_FIELD(hv, conf, max_node_fail_requeue, uint32_t, TRUE); + FETCH_FIELD(hv, conf, max_preempt_requeue, uint32_t, TRUE); FETCH_FIELD(hv, conf, max_dbd_msgs, uint32_t, TRUE); FETCH_FIELD(hv, conf, max_job_cnt, uint32_t, TRUE); FETCH_FIELD(hv, conf, max_job_id, uint32_t, FALSE); diff --git a/doc/man/man5/slurm.conf.5 b/doc/man/man5/slurm.conf.5 index 5c339de1344..5cec172af84 100644 --- a/doc/man/man5/slurm.conf.5 +++ b/doc/man/man5/slurm.conf.5 @@ -2402,12 +2402,28 @@ The value may not exceed 65533. .TP \fBMaxBatchRequeue\fR -Maximum number of times a batch job may be automatically requeued before -being marked as JobHeldAdmin. (Mainly useful when the \fBSchedulerParameters\fR -option \fBnohold_on_prolog_fail\fR is enabled.) +Maximum number of times a batch job may be requeued due to launch or +prolog failures before being held. (Mainly useful when the +\fBSchedulerParameters\fR option \fBnohold_on_prolog_fail\fR is enabled.) The default value is 5. .IP +.TP +\fBMaxNodeFailRequeue\fR +Maximum number of times a batch job may be requeued after an allocated +node fails before the job is held. This can limit jobs suspected of repeatedly +triggering node failures from being requeued indefinitely; Slurm does not +diagnose whether the job or node caused the failure. A value of 0 disables the +limit. The default value is 5. +.IP + +.TP +\fBMaxPreemptRequeue\fR +Maximum number of times a batch job may be requeued due to preemption or +operator requeue before the job is held. A value of 0 disables the limit. +The default value is 0. +.IP + .TP \fBNodeFeaturesPlugins\fR Identifies the plugins to be used for support of node features which can diff --git a/slurm/slurm.h b/slurm/slurm.h index a0343de1f6e..bf53a4c417f 100644 --- a/slurm/slurm.h +++ b/slurm/slurm.h @@ -658,6 +658,8 @@ enum job_state_reason { * (Unknown) */ WAIT_MAX_POWERED_NODES, /* max_powered_nodes reached */ WAIT_MPI_PORTS_BUSY, /* MPI resv_ports busy */ + WAIT_MAX_NODE_FAIL_REQUEUE, /* MaxNodeFailRequeue reached */ + WAIT_MAX_PREEMPT_REQUEUE, /* MaxPreemptRequeue reached */ REASON_END, /* end of table */ }; @@ -3145,7 +3147,12 @@ typedef struct { char *mail_domain; /* default domain to append to usernames */ char *mail_prog; /* pathname of mail program */ uint32_t max_array_sz; /* Maximum job array size */ - uint32_t max_batch_requeue; /* maximum number of requeues */ + uint32_t max_batch_requeue; /* maximum number of launch-failure + * requeues */ + uint32_t max_node_fail_requeue; /* maximum number of node-failure + * requeues */ + uint32_t max_preempt_requeue; /* maximum number of preemption/operator + * requeues */ uint32_t max_dbd_msgs; /* maximum number of messages queued while DBD * is not connected */ uint32_t max_job_cnt; /* maximum number of active jobs */ diff --git a/src/api/config_info.c b/src/api/config_info.c index ac9772f8c85..d292d49d413 100644 --- a/src/api/config_info.c +++ b/src/api/config_info.c @@ -853,6 +853,12 @@ extern void *slurm_ctl_conf_2_key_pairs(slurm_conf_t *conf) add_key_pair(ret_list, "MaxBatchRequeue", "%u", conf->max_batch_requeue); + add_key_pair(ret_list, "MaxNodeFailRequeue", "%u", + conf->max_node_fail_requeue); + + add_key_pair(ret_list, "MaxPreemptRequeue", "%u", + conf->max_preempt_requeue); + add_key_pair(ret_list, "MaxDBDMsgs", "%u", conf->max_dbd_msgs); add_key_pair(ret_list, "MaxJobCount", "%u", conf->max_job_cnt); diff --git a/src/common/job_record.c b/src/common/job_record.c index 70b7818c27f..b08f83fcfb7 100644 --- a/src/common/job_record.c +++ b/src/common/job_record.c @@ -2198,6 +2198,11 @@ extern void job_record_pack_common(job_record_t *dump_job_ptr, pack32(dump_job_ptr->req_switch, buffer); pack_time(dump_job_ptr->resize_time, buffer); pack16(dump_job_ptr->restart_cnt, buffer); + if (for_state && + (protocol_version >= SLURM_26_11_PROTOCOL_VERSION)) { + pack32(dump_job_ptr->node_fail_requeue_cnt, buffer); + pack32(dump_job_ptr->preempt_requeue_cnt, buffer); + } packstr(dump_job_ptr->resv_name, buffer); packstr(dump_job_ptr->resv_ports, buffer); @@ -2483,6 +2488,10 @@ extern int job_record_unpack_common(job_record_t *job_ptr, safe_unpack32(&job_ptr->req_switch, buffer); safe_unpack_time(&job_ptr->resize_time, buffer); safe_unpack16(&job_ptr->restart_cnt, buffer); + if (protocol_version >= SLURM_26_11_PROTOCOL_VERSION) { + safe_unpack32(&job_ptr->node_fail_requeue_cnt, buffer); + safe_unpack32(&job_ptr->preempt_requeue_cnt, buffer); + } safe_unpackstr(&job_ptr->resv_name, buffer); safe_unpackstr(&job_ptr->resv_ports, buffer); diff --git a/src/common/job_record.h b/src/common/job_record.h index 657b09d956b..12824466509 100644 --- a/src/common/job_record.h +++ b/src/common/job_record.h @@ -471,6 +471,8 @@ struct job_record { slurmdb_qos_rec_t *qos_blocking_ptr; /* internal use only, DON'T PACK */ uint8_t reboot; /* node reboot requested before start */ uint16_t restart_cnt; /* count of restarts */ + uint32_t node_fail_requeue_cnt; /* count of node-failure requeues */ + uint32_t preempt_requeue_cnt; /* count of preemption/operator requeues */ time_t resize_time; /* time of latest size change */ uint32_t resv_id; /* reservation ID */ list_t *resv_list; /* Filled in if the job is requesting diff --git a/src/common/job_state_reason.c b/src/common/job_state_reason.c index f2c092e4e02..345623238c2 100644 --- a/src/common/job_state_reason.c +++ b/src/common/job_state_reason.c @@ -306,6 +306,7 @@ const static entry_t jsra[] = { .str = "AssocMaxSubmitJobLimit", }, [WAIT_MAX_REQUEUE] = { + /* Keep legacy spelling for backward compatibility. */ .str = "JobHoldMaxRequeue", }, [WAIT_ARRAY_TASK_LIMIT] = { @@ -882,6 +883,12 @@ const static entry_t jsra[] = { [WAIT_MPI_PORTS_BUSY] = { .str = "MpiPortsBusy", }, + [WAIT_MAX_NODE_FAIL_REQUEUE] = { + .str = "JobHeldMaxNodeFailRequeue", + }, + [WAIT_MAX_PREEMPT_REQUEUE] = { + .str = "JobHeldMaxPreemptRequeue", + }, }; extern const char *job_state_reason_string(enum job_state_reason inx) diff --git a/src/common/read_config.c b/src/common/read_config.c index 332a5fd6e5b..551a8981979 100644 --- a/src/common/read_config.c +++ b/src/common/read_config.c @@ -322,6 +322,8 @@ s_p_options_t slurm_conf_options[] = { {"MailProg", S_P_STRING}, {"MaxArraySize", S_P_UINT32}, {"MaxBatchRequeue", S_P_UINT32}, + {"MaxNodeFailRequeue", S_P_UINT32}, + {"MaxPreemptRequeue", S_P_UINT32}, {"MaxDBDMsgs", S_P_UINT32}, {"MaxJobCount", S_P_UINT32}, {"MaxJobId", S_P_UINT32}, @@ -2970,6 +2972,8 @@ extern void init_slurm_conf(slurm_conf_t *conf) xfree(conf->mail_prog); conf->max_array_sz = NO_VAL; conf->max_batch_requeue = NO_VAL; + conf->max_node_fail_requeue = NO_VAL; + conf->max_preempt_requeue = NO_VAL; conf->max_dbd_msgs = 0; conf->max_job_cnt = NO_VAL; conf->max_job_id = NO_VAL; @@ -4397,6 +4401,14 @@ static int _validate_and_set_defaults(slurm_conf_t *conf, hashtbl)) conf->max_batch_requeue = DEFAULT_MAX_BATCH_REQUEUE; + if (!s_p_get_uint32(&conf->max_node_fail_requeue, "MaxNodeFailRequeue", + hashtbl)) + conf->max_node_fail_requeue = DEFAULT_MAX_NODE_FAIL_REQUEUE; + + if (!s_p_get_uint32(&conf->max_preempt_requeue, "MaxPreemptRequeue", + hashtbl)) + conf->max_preempt_requeue = DEFAULT_MAX_PREEMPT_REQUEUE; + if (!s_p_get_uint32(&conf->max_dbd_msgs, "MaxDBDMsgs", hashtbl)) conf->max_dbd_msgs = 0; else if (conf->max_dbd_msgs < DEFAULT_MAX_DBD_MSGS) { diff --git a/src/common/read_config.h b/src/common/read_config.h index 5a7a85ff8df..5ec23a1967a 100644 --- a/src/common/read_config.h +++ b/src/common/read_config.h @@ -119,6 +119,8 @@ typedef struct node_record node_record_t; #define DEFAULT_MAIL_PROG_ALT "/usr/bin/mail" #define DEFAULT_MAX_ARRAY_SIZE 1001 #define DEFAULT_MAX_BATCH_REQUEUE 5 +#define DEFAULT_MAX_NODE_FAIL_REQUEUE 5 +#define DEFAULT_MAX_PREEMPT_REQUEUE 0 #define DEFAULT_MAX_DBD_MSGS 10000 #define DEFAULT_MAX_JOB_COUNT 10000 #define DEFAULT_MAX_JOB_ID 0x03ff0000 diff --git a/src/common/slurm_protocol_pack.c b/src/common/slurm_protocol_pack.c index 23fdfd81e40..888304b6b67 100644 --- a/src/common/slurm_protocol_pack.c +++ b/src/common/slurm_protocol_pack.c @@ -3971,6 +3971,10 @@ static void _pack_slurm_conf(const slurm_conf_t *conf, pack32(conf->max_array_sz, buffer); pack32(conf->max_batch_requeue, buffer); + if (protocol_version >= SLURM_26_11_PROTOCOL_VERSION) { + pack32(conf->max_node_fail_requeue, buffer); + pack32(conf->max_preempt_requeue, buffer); + } pack32(conf->max_dbd_msgs, buffer); packstr(conf->mail_domain, buffer); packstr(conf->mail_prog, buffer); @@ -4789,6 +4793,10 @@ static int _unpack_slurm_conf(slurm_conf_t **conf_ptr, safe_unpack32(&conf->max_array_sz, buffer); safe_unpack32(&conf->max_batch_requeue, buffer); + if (protocol_version >= SLURM_26_11_PROTOCOL_VERSION) { + safe_unpack32(&conf->max_node_fail_requeue, buffer); + safe_unpack32(&conf->max_preempt_requeue, buffer); + } safe_unpack32(&conf->max_dbd_msgs, buffer); safe_unpackstr(&conf->mail_domain, buffer); safe_unpackstr(&conf->mail_prog, buffer); @@ -5521,6 +5529,11 @@ static int _unpack_slurm_conf(slurm_conf_t **conf_ptr, safe_unpackstr(&conf->x11_params, buffer); } + if (protocol_version < SLURM_26_11_PROTOCOL_VERSION) { + conf->max_node_fail_requeue = DEFAULT_MAX_NODE_FAIL_REQUEUE; + conf->max_preempt_requeue = DEFAULT_MAX_PREEMPT_REQUEUE; + } + *conf_ptr = conf; return SLURM_SUCCESS; diff --git a/src/plugins/data_parser/v0.0.46/parsers.c b/src/plugins/data_parser/v0.0.46/parsers.c index 744cb6a0fc7..21ecbb17d54 100644 --- a/src/plugins/data_parser/v0.0.46/parsers.c +++ b/src/plugins/data_parser/v0.0.46/parsers.c @@ -12442,6 +12442,8 @@ static const parser_t PARSER_ARRAY(SLURM_CONF)[] = { add_parse(STRING, mail_prog, "MailProg", "Pathname of mail program"), add_parse(UINT32, max_array_sz, "MaxArraySize", "Maximum job array size"), add_parse(UINT32, max_batch_requeue, "MaxBatchRequeue", "Max times a batch job may be auto-requeued before being held"), + add_parse(UINT32, max_node_fail_requeue, "MaxNodeFailRequeue", "Max node-failure requeues before hold (0=disabled)"), + add_parse(UINT32, max_preempt_requeue, "MaxPreemptRequeue", "Max preemption/operator requeues before hold (0=disabled)"), add_parse(UINT32, max_dbd_msgs, "MaxDBDMsgs", "Maximum number of messages queued while DBD is not connected"), add_parse(UINT32, max_job_cnt, "MaxJobCount", "Maximum number of active jobs"), add_parse(UINT32, max_job_id, "MaxJobId", "Maximum job id before wrapping back to FirstJobId"), @@ -12710,6 +12712,8 @@ static const parser_t PARSER_ARRAY(SLURM_CONF_META)[] = { add_skip(mail_prog), add_skip(max_array_sz), add_skip(max_batch_requeue), + add_skip(max_node_fail_requeue), + add_skip(max_preempt_requeue), add_skip(max_dbd_msgs), add_skip(max_job_cnt), add_skip(max_job_id), diff --git a/src/slurmctld/job_mgr.c b/src/slurmctld/job_mgr.c index eef5cd72e60..5ad4f50ea3e 100644 --- a/src/slurmctld/job_mgr.c +++ b/src/slurmctld/job_mgr.c @@ -365,7 +365,23 @@ static void _signal_pending_job_array_tasks(job_record_t *job_ptr, static void _add_job_hash(job_record_t *job_ptr); static void _add_job_hash_sluid(job_record_t *job_ptr); static void _add_job_array_hash(job_record_t *job_ptr); -static void _handle_requeue_limit(job_record_t *job_ptr, const char *caller); +static void _handle_max_batch_requeue_limit(job_record_t *job_ptr, + const char *caller); +/* + * Requeue cause — each maps to its own independent counter: + * JOB_LAUNCH_FAILURE → job launch or prolog errors → batch_requeue_cnt + * NODE_FAIL → allocated node died → node_fail_requeue_cnt + * PREEMPT → preempted by higher-pri job → preempt_requeue_cnt + * OPERATOR → admin scontrol requeue → preempt_requeue_cnt + */ +typedef enum { + REQUEUE_CAUSE_JOB_LAUNCH_FAILURE, + REQUEUE_CAUSE_NODE_FAIL, + REQUEUE_CAUSE_PREEMPT, + REQUEUE_CAUSE_OPERATOR, +} requeue_cause_t; +static void _handle_requeue_limits(job_record_t *job_ptr, requeue_cause_t cause, + const char *caller); static int _copy_job_desc_to_file(job_desc_msg_t * job_desc, uint32_t job_id); static int _copy_job_desc_to_job_record(job_desc_msg_t * job_desc, @@ -3081,6 +3097,10 @@ static int _foreach_kill_running_job_by_node(void *x, void *arg) */ acct_policy_add_job_submit(job_ptr, false); + _handle_requeue_limits(job_ptr, + REQUEUE_CAUSE_NODE_FAIL, + __func__); + if (!job_ptr->node_bitmap_cg || bit_ffs(job_ptr->node_bitmap_cg) == -1) batch_requeue_fini(job_ptr); @@ -6096,7 +6116,8 @@ extern int prolog_complete(prolog_complete_msg_t *msg) return SLURM_SUCCESS; } -static void _handle_requeue_limit(job_record_t *job_ptr, const char *caller) +static void _handle_max_batch_requeue_limit(job_record_t *job_ptr, + const char *caller) { if (job_ptr->batch_flag <= slurm_conf.max_batch_requeue) return; @@ -6113,6 +6134,53 @@ static void _handle_requeue_limit(job_record_t *job_ptr, const char *caller) job_ptr->priority = 0; } +static void _hold_requeue_limit(job_record_t *job_ptr, + enum job_state_reason reason, + const char *state_desc, const char *caller) +{ + debug("%s: Holding %pJ, %s", caller, job_ptr, state_desc); + job_state_set_flag(job_ptr, JOB_REQUEUE_HOLD); + job_ptr->state_reason = reason; + xfree(job_ptr->state_desc); + job_ptr->state_desc = xstrdup(state_desc); + job_ptr->priority = 0; +} + +static void _handle_requeue_limits(job_record_t *job_ptr, + requeue_cause_t cause, + const char *caller) +{ + switch (cause) { + case REQUEUE_CAUSE_NODE_FAIL: + ++job_ptr->node_fail_requeue_cnt; + /* Hold only when limit is set (> 0) and exceeded */ + if ((slurm_conf.max_node_fail_requeue > 0) && + (job_ptr->node_fail_requeue_cnt > + slurm_conf.max_node_fail_requeue)) + _hold_requeue_limit( + job_ptr, WAIT_MAX_NODE_FAIL_REQUEUE, + "node failure requeue limit exceeded", + caller); + break; + case REQUEUE_CAUSE_PREEMPT: + case REQUEUE_CAUSE_OPERATOR: + ++job_ptr->preempt_requeue_cnt; + /* Hold only when limit is set (> 0) and exceeded */ + if ((slurm_conf.max_preempt_requeue > 0) && + (job_ptr->preempt_requeue_cnt > + slurm_conf.max_preempt_requeue)) + _hold_requeue_limit( + job_ptr, WAIT_MAX_PREEMPT_REQUEUE, + "preemption requeue limit exceeded", + caller); + break; + case REQUEUE_CAUSE_JOB_LAUNCH_FAILURE: + default: + _handle_max_batch_requeue_limit(job_ptr, caller); + break; + } +} + static int _job_complete(job_record_t *job_ptr, uid_t uid, bool requeue, bool node_fail, uint32_t job_return_code) { @@ -6191,13 +6259,17 @@ static int _job_complete(job_record_t *job_ptr, uid_t uid, bool requeue, * accounting logs. Set a new submit time so the restarted * job looks like a new job. */ + requeue_cause_t requeue_cause; job_ptr->end_time = now; if (job_ptr->bit_flags & GRACE_PREEMPT) { + requeue_cause = REQUEUE_CAUSE_PREEMPT; job_state_set(job_ptr, (JOB_PREEMPTED | job_comp_flag)); /* clear signal sent on GracePeriod start */ job_ptr->bit_flags &= (~GRACE_PREEMPT); } else { + requeue_cause = node_fail ? REQUEUE_CAUSE_NODE_FAIL : + REQUEUE_CAUSE_JOB_LAUNCH_FAILURE; job_state_set(job_ptr, JOB_NODE_FAIL); job_ptr->exit_code = job_return_code; } @@ -6217,7 +6289,7 @@ static int _job_complete(job_record_t *job_ptr, uid_t uid, bool requeue, use_cloud = true; } } - if (!use_cloud) + if (!use_cloud && (requeue_cause == REQUEUE_CAUSE_JOB_LAUNCH_FAILURE)) job_ptr->batch_flag++; /* only one retry */ job_ptr->restart_cnt++; @@ -6242,8 +6314,8 @@ static int _job_complete(job_record_t *job_ptr, uid_t uid, bool requeue, info("%s: requeue %pJ per user/system request", __func__, job_ptr); } - /* hold job if over requeue limit */ - _handle_requeue_limit(job_ptr, __func__); + /* hold job if over the per-cause requeue limit */ + _handle_requeue_limits(job_ptr, requeue_cause, __func__); } else if (IS_JOB_PENDING(job_ptr) && job_ptr->details && job_ptr->batch_flag) { /* @@ -11959,6 +12031,8 @@ static bool _top_priority(job_record_t *job_ptr, uint32_t het_job_offset) && (job_ptr->state_reason != FAIL_QOS) && (job_ptr->state_reason != WAIT_HELD) && (job_ptr->state_reason != WAIT_HELD_USER) + && (job_ptr->state_reason != WAIT_MAX_NODE_FAIL_REQUEUE) + && (job_ptr->state_reason != WAIT_MAX_PREEMPT_REQUEUE) && job_ptr->state_reason != WAIT_MAX_REQUEUE) { job_ptr->state_reason = WAIT_HELD; xfree(job_ptr->state_desc); @@ -12060,6 +12134,8 @@ static void _release_job_rec(job_record_t *job_ptr, uid_t uid) job_state_unset_flag(job_ptr, JOB_SPECIAL_EXIT); xfree(job_ptr->state_desc); job_ptr->exit_code = 0; + job_ptr->node_fail_requeue_cnt = 0; + job_ptr->preempt_requeue_cnt = 0; if (job_ptr->licenses && !job_ptr->license_list) { /* @@ -14989,6 +15065,8 @@ static int _update_job(job_record_t *job_ptr, job_desc_msg_t *job_desc, * could launch before the extern step sets up x11. */ && (job_ptr->state_reason != WAIT_PROLOG) + && (job_ptr->state_reason != WAIT_MAX_NODE_FAIL_REQUEUE) + && (job_ptr->state_reason != WAIT_MAX_PREEMPT_REQUEUE) && (job_ptr->state_reason != WAIT_MAX_REQUEUE)) { job_ptr->state_reason = WAIT_NO_REASON; xfree(job_ptr->state_desc); @@ -17037,6 +17115,8 @@ extern bool job_independent(job_record_t *job_ptr) (job_ptr->state_reason == WAIT_HELD) || (job_ptr->state_reason == WAIT_HELD_USER) || (job_ptr->state_reason == WAIT_MAX_REQUEUE) || + (job_ptr->state_reason == WAIT_MAX_NODE_FAIL_REQUEUE) || + (job_ptr->state_reason == WAIT_MAX_PREEMPT_REQUEUE) || (job_ptr->state_reason == WAIT_RESV_DELETED) || (job_ptr->state_reason == WAIT_RESV_INVALID) || (job_ptr->state_reason == WAIT_DEP_INVALID)) @@ -17723,6 +17803,7 @@ static int _job_requeue_op(uid_t uid, job_record_t *job_ptr, bool preempt, bool is_running = false, is_suspended = false, is_completed = false; bool is_completing = false; bool requeue_fini_called = false; + bool count_operator_requeue = true; bool force_requeue = false; time_t now = time(NULL); uint32_t completing_flags = 0; @@ -17935,6 +18016,7 @@ static int _job_requeue_op(uid_t uid, job_record_t *job_ptr, bool preempt, xstrdup("job requeued in special exit state"); debug("%s: Holding %pJ, special exit", __func__, job_ptr); job_ptr->priority = 0; + count_operator_requeue = false; } if (flags & JOB_REQUEUE_HOLD) { job_ptr->state_reason = WAIT_HELD_USER; @@ -17942,10 +18024,13 @@ static int _job_requeue_op(uid_t uid, job_record_t *job_ptr, bool preempt, job_ptr->state_desc = xstrdup("job requeued in held state"); debug("%s: Holding %pJ, requeue-hold exit", __func__, job_ptr); job_ptr->priority = 0; + count_operator_requeue = false; } - if (flags & JOB_LAUNCH_FAILED) { + if (preempt) { + _handle_requeue_limits(job_ptr, REQUEUE_CAUSE_PREEMPT, __func__); + } else if (flags & JOB_LAUNCH_FAILED) { job_ptr->batch_flag++; - _handle_requeue_limit(job_ptr, __func__); + _handle_requeue_limits(job_ptr, REQUEUE_CAUSE_JOB_LAUNCH_FAILURE, __func__); /* If job not already held, make it so if needed. */ if (!(job_ptr->job_state & JOB_REQUEUE_HOLD) && @@ -17965,6 +18050,9 @@ static int _job_requeue_op(uid_t uid, job_record_t *job_ptr, bool preempt, } job_ptr->priority = 0; } + } else if (count_operator_requeue && validate_operator(uid)) { + _handle_requeue_limits(job_ptr, REQUEUE_CAUSE_OPERATOR, + __func__); } /* diff --git a/src/slurmctld/job_scheduler.c b/src/slurmctld/job_scheduler.c index 43b7b63d725..8f1d6aba92f 100644 --- a/src/slurmctld/job_scheduler.c +++ b/src/slurmctld/job_scheduler.c @@ -365,6 +365,8 @@ static bool _job_runnable_test1(job_record_t *job_ptr, bool sched_plugin) (job_ptr->state_reason != WAIT_HELD) && (job_ptr->state_reason != WAIT_HELD_USER) && (job_ptr->state_reason != WAIT_MAX_REQUEUE) && + (job_ptr->state_reason != WAIT_MAX_NODE_FAIL_REQUEUE) && + (job_ptr->state_reason != WAIT_MAX_PREEMPT_REQUEUE) && (job_ptr->state_reason != WAIT_RESV_INVALID) && (job_ptr->state_reason != WAIT_RESV_DELETED)) { job_ptr->state_reason = WAIT_HELD; diff --git a/src/slurmctld/node_scheduler.c b/src/slurmctld/node_scheduler.c index 5d71a5371b3..0e1fabde96e 100644 --- a/src/slurmctld/node_scheduler.c +++ b/src/slurmctld/node_scheduler.c @@ -2634,7 +2634,9 @@ extern int select_nodes(job_node_select_t *job_node_select, && (job_ptr->state_reason != FAIL_BURST_BUFFER_OP) && (job_ptr->state_reason != WAIT_HELD) && (job_ptr->state_reason != WAIT_HELD_USER) - && (job_ptr->state_reason != WAIT_MAX_REQUEUE)) { + && (job_ptr->state_reason != WAIT_MAX_REQUEUE) + && (job_ptr->state_reason != WAIT_MAX_NODE_FAIL_REQUEUE) + && (job_ptr->state_reason != WAIT_MAX_PREEMPT_REQUEUE)) { job_ptr->state_reason = WAIT_HELD; } return ESLURM_JOB_HELD; diff --git a/src/slurmctld/proc_req.c b/src/slurmctld/proc_req.c index 988fc6886d0..1c6a329deba 100644 --- a/src/slurmctld/proc_req.c +++ b/src/slurmctld/proc_req.c @@ -407,6 +407,8 @@ static void _fill_ctld_conf(slurm_conf_t *conf_ptr) conf_ptr->mail_prog = xstrdup(conf->mail_prog); conf_ptr->max_array_sz = conf->max_array_sz; conf_ptr->max_batch_requeue = conf->max_batch_requeue; + conf_ptr->max_node_fail_requeue = conf->max_node_fail_requeue; + conf_ptr->max_preempt_requeue = conf->max_preempt_requeue; conf_ptr->max_dbd_msgs = conf->max_dbd_msgs; conf_ptr->max_job_cnt = conf->max_job_cnt; conf_ptr->max_job_id = conf->max_job_id; diff --git a/src/slurmd/common/slurmstepd_init.c b/src/slurmd/common/slurmstepd_init.c index fb56045e6db..91837832620 100644 --- a/src/slurmd/common/slurmstepd_init.c +++ b/src/slurmd/common/slurmstepd_init.c @@ -228,6 +228,8 @@ extern void pack_slurm_conf_lite(buf_t *buffer) /* mail_prog */ /* max_array_sz */ /* max_batch_requeue */ + /* max_node_fail_requeue */ + /* max_preempt_requeue */ /* max_dbd_msgs */ /* max_job_cnt */ /* max_job_id */ @@ -464,6 +466,8 @@ extern int unpack_slurm_conf_lite_no_alloc(buf_t *buffer) /* mail_prog */ /* max_array_sz */ /* max_batch_requeue */ + /* max_node_fail_requeue */ + /* max_preempt_requeue */ /* max_dbd_msgs */ /* max_job_cnt */ /* max_job_id */ diff --git a/testsuite/python/tests/test_105_11.py b/testsuite/python/tests/test_105_11.py new file mode 100644 index 00000000000..69ac71e6f38 --- /dev/null +++ b/testsuite/python/tests/test_105_11.py @@ -0,0 +1,659 @@ +############################################################################ +# Copyright (C) SchedMD LLC. +############################################################################ +""" +Cause-aware batch job requeue limits (MaxNodeFailRequeue, MaxPreemptRequeue). + +Verifies that node-failure requeues, preemption requeues, and launch-failure +requeues are counted independently, each against its own configurable limit. + +Tested: +- Node-failure requeues are held at MaxNodeFailRequeue with + JobHeldMaxNodeFailRequeue reason. +- Preemption requeues are held at MaxPreemptRequeue with + JobHeldMaxPreemptRequeue reason. +- Node-failure requeues do NOT count toward MaxBatchRequeue. +- Node-failure requeues do NOT count toward MaxPreemptRequeue. +- Preemption requeues do NOT count toward MaxNodeFailRequeue. +- scontrol requeuehold does NOT consume the preemption budget. +- User-initiated scontrol requeue does NOT consume the preemption budget. +- Operator-initiated scontrol requeue consumes the preemption budget. +- scontrol release resets per-cause counters (fresh allowance). +- MaxPreemptRequeue=0 (the default) means unlimited; a job is never held for + preemption unless an admin sets a positive limit. +""" +import atf +import pytest + +NODE_FAIL_LIMIT = 2 +PREEMPT_LIMIT = 2 +QOS_LOW = "test_requeue_low" +QOS_HIGH = "test_requeue_high" + +pytestmark = pytest.mark.slow + + +def _assert_requeue_hold(job_id, reason_markers, context): + job_state = atf.get_job_parameter(job_id, "JobState") + reason = atf.get_job_parameter(job_id, "Reason") or "" + priority = atf.get_job_parameter(job_id, "Priority") + + assert job_state == "REQUEUE_HOLD", ( + f"{context}: expected JobState=REQUEUE_HOLD, got {job_state}" + ) + assert any(marker in reason for marker in reason_markers) or any( + marker.lower() in reason.lower() for marker in reason_markers + ), f"{context}: unexpected hold reason {reason}" + assert str(priority) == "0", ( + f"{context}: held job should have Priority=0, got {priority}" + ) + + return reason + + +@pytest.fixture(scope="module", autouse=True) +def setup(): + atf.require_accounting(modify=True) + atf.require_config_parameter_includes("AccountingStorageEnforce", "qos") + atf.require_config_parameter_includes( + "AccountingStorageEnforce", "associations" + ) + atf.require_nodes(2, [("CPUs", 2)]) + atf.require_config_parameter("SelectType", "select/cons_tres") + atf.require_config_parameter("PreemptType", "preempt/qos") + atf.require_config_parameter("PreemptMode", "REQUEUE") + atf.require_config_parameter("MaxNodeFailRequeue", str(NODE_FAIL_LIMIT)) + # MaxPreemptRequeue defaults to 0 (unlimited); pin a small positive limit + # here so the hold-at-limit tests can exercise the preemption path. + atf.require_config_parameter("MaxPreemptRequeue", str(PREEMPT_LIMIT)) + atf.require_config_parameter("MaxBatchRequeue", "5") + atf.require_config_parameter_includes( + "SchedulerParameters", "requeue_delay=0" + ) + atf.require_config_parameter("ReturnToService", "1") + atf.require_slurm_running() + + cluster = atf.get_config_parameter("ClusterName") + user = atf.properties["test-user"] + su = atf.properties["slurm-user"] + qos_list = f"normal,{QOS_LOW},{QOS_HIGH}" + + atf.run_command( + f"sacctmgr -i add qos {QOS_LOW} 2>/dev/null || true", + user=su, + fatal=True, + ) + atf.run_command( + f"sacctmgr -i add qos {QOS_HIGH} Preempt={QOS_LOW} " + f"2>/dev/null || sacctmgr -i modify qos {QOS_HIGH} " + f"set Preempt={QOS_LOW}", + user=su, + fatal=True, + ) + atf.run_command( + f"sacctmgr -i add user {user} cluster={cluster} account=root " + f"qos={qos_list} 2>/dev/null || sacctmgr -i modify user {user} " + f"where cluster={cluster} set qos={qos_list}", + user=su, + fatal=True, + ) + + yield + + atf.run_command( + f"sacctmgr -i modify user {user} where cluster={cluster} " + f"set qos=normal", + user=su, + quiet=True, + ) + atf.run_command( + f"sacctmgr -i remove qos {QOS_HIGH},{QOS_LOW}", + user=su, + quiet=True, + ) + + +@pytest.fixture(scope="module") +def nodes(): + node_list = list(atf.nodes) + assert len(node_list) >= 2, "Need at least 2 nodes" + return node_list[0], node_list[1] + + +@pytest.fixture(scope="function", autouse=True) +def cleanup_nodes(nodes): + """Resume any downed nodes after each test.""" + yield + for node in nodes: + atf.run_command( + f"scontrol update nodename={node} state=RESUME", + user=atf.properties["slurm-user"], + quiet=True, + ) + atf.wait_for_node_state(node, "IDLE", timeout=30, fatal=False) + + +def test_node_fail_hold_at_limit(nodes): + """A job requeued by node failure is held after MaxNodeFailRequeue.""" + target_node, other_node = nodes + + # Pin a filler job on the other node so our test job only runs on target + filler = atf.submit_job_sbatch( + f"-N1 -w {other_node} --exclusive --requeue --wrap 'sleep 600'", + fatal=True, + ) + atf.wait_for_job_state(filler, "RUNNING", fatal=True) + + # Submit test job pinned to target node + job_id = atf.submit_job_sbatch( + f"-N1 -w {target_node} --exclusive --requeue --wrap 'sleep 600'", + fatal=True, + ) + atf.wait_for_job_state(job_id, "RUNNING", fatal=True) + + # Cycle through node failures up to and past the limit + for cycle in range(1, NODE_FAIL_LIMIT + 2): + atf.run_command( + f"scontrol update nodename={target_node} state=DOWN reason=test_cycle_{cycle}", + user=atf.properties["slurm-user"], + fatal=True, + ) + # Wait for job to leave RUNNING + atf.repeat_until( + lambda: atf.get_job_parameter(job_id, "JobState"), + lambda s: s != "RUNNING", + timeout=30, + fatal=True, + ) + + if cycle <= NODE_FAIL_LIMIT: + # Should NOT be held yet + reason = atf.get_job_parameter(job_id, "Reason") or "" + assert "MaxNodeFailRequeue" not in reason, ( + f"Job should not be held after {cycle} node failures " + f"(limit is {NODE_FAIL_LIMIT}), Reason={reason}" + ) + # Resume node for next cycle + atf.run_command( + f"scontrol update nodename={target_node} state=RESUME", + user=atf.properties["slurm-user"], + fatal=True, + ) + atf.wait_for_node_state(target_node, "IDLE", fatal=True) + atf.wait_for_job_state(job_id, "RUNNING", fatal=True) + else: + # Past the limit - job must be held + _assert_requeue_hold( + job_id, + ["node_failure_requeue_limit", "JobHeldMaxNodeFailRequeue"], + f"Job after {cycle} node failures", + ) + + atf.cancel_jobs([filler, job_id]) + + +def test_node_fail_does_not_trip_other_requeue_limits(nodes): + """Node-failure requeues must NOT count toward other requeue limits.""" + target_node, other_node = nodes + filler = None + job_id = None + max_batch_requeue = 1 + max_preempt_requeue = 1 + max_node_fail_requeue = max_batch_requeue + 2 + node_fail_cycles = max_batch_requeue + 1 + + try: + atf.set_config_parameter("MaxBatchRequeue", str(max_batch_requeue)) + atf.set_config_parameter("MaxPreemptRequeue", str(max_preempt_requeue)) + atf.set_config_parameter( + "MaxNodeFailRequeue", str(max_node_fail_requeue) + ) + atf.run_command( + "scontrol reconfigure", + user=atf.properties["slurm-user"], + fatal=True, + ) + + filler = atf.submit_job_sbatch( + f"-N1 -w {other_node} --exclusive --requeue --wrap 'sleep 600'", + fatal=True, + ) + atf.wait_for_job_state(filler, "RUNNING", fatal=True) + + job_id = atf.submit_job_sbatch( + f"-N1 -w {target_node} --exclusive --requeue --wrap 'sleep 600'", + fatal=True, + ) + atf.wait_for_job_state(job_id, "RUNNING", fatal=True) + + for cycle in range(1, node_fail_cycles + 1): + atf.run_command( + f"scontrol update nodename={target_node} state=DOWN " + f"reason=batch_independent_{cycle}", + user=atf.properties["slurm-user"], + fatal=True, + ) + atf.repeat_until( + lambda: atf.get_job_parameter(job_id, "JobState"), + lambda s: s != "RUNNING", + timeout=30, + fatal=True, + ) + + reason = atf.get_job_parameter(job_id, "Reason") or "" + assert "MaxRequeue" not in reason, ( + f"Node-failure requeue should not trigger MaxBatchRequeue " + f"after {cycle} node failures, Reason={reason}" + ) + assert "MaxPreemptRequeue" not in reason, ( + f"Node-failure requeue should not trigger " + f"MaxPreemptRequeue after {cycle} node failures, " + f"Reason={reason}" + ) + assert "preemption_requeue_limit" not in reason.lower(), ( + f"Node-failure requeue should not trigger preemption hold " + f"after {cycle} node failures, Reason={reason}" + ) + assert "MaxNodeFail" not in reason, ( + f"Node-failure requeue should not hit MaxNodeFailRequeue " + f"before {max_node_fail_requeue + 1} failures, " + f"Reason={reason}" + ) + + atf.run_command( + f"scontrol update nodename={target_node} state=RESUME", + user=atf.properties["slurm-user"], + fatal=True, + ) + atf.wait_for_node_state(target_node, "IDLE", fatal=True) + atf.wait_for_job_state(job_id, "RUNNING", fatal=True) + finally: + job_ids = [job for job in [filler, job_id] if job] + if job_ids: + atf.cancel_jobs(job_ids) + + atf.set_config_parameter("MaxBatchRequeue", "5") + atf.set_config_parameter("MaxNodeFailRequeue", str(NODE_FAIL_LIMIT)) + atf.set_config_parameter("MaxPreemptRequeue", str(PREEMPT_LIMIT)) + atf.run_command( + "scontrol reconfigure", + user=atf.properties["slurm-user"], + fatal=True, + ) + + +def test_preempt_hold_at_limit(nodes): + """A job preempted past MaxPreemptRequeue is held.""" + target_node = nodes[0] + + for cycle in range(1, PREEMPT_LIMIT + 2): + # Submit low-priority victim + if cycle == 1: + victim = atf.submit_job_sbatch( + f"-N1 -w {target_node} --exclusive --requeue " + f"--qos={QOS_LOW} --wrap 'sleep 600'", + fatal=True, + ) + atf.wait_for_job_state(victim, "RUNNING", fatal=True, timeout=60) + + # Submit high-priority preemptor + preemptor = atf.submit_job_sbatch( + f"-N1 -w {target_node} --exclusive " + f"--qos={QOS_HIGH} --wrap 'sleep 15'", + fatal=True, + ) + atf.wait_for_job_state(preemptor, "RUNNING", fatal=True) + + # Wait for preemptor to finish + atf.wait_for_job_state(preemptor, "DONE", fatal=True, timeout=30) + + if cycle <= PREEMPT_LIMIT: + reason = atf.get_job_parameter(victim, "Reason") or "" + assert "MaxPreemptRequeue" not in reason, ( + f"Victim should not be held after {cycle} preemptions " + f"(limit is {PREEMPT_LIMIT}), Reason={reason}" + ) + else: + _assert_requeue_hold( + victim, + ["preemption_requeue_limit", "JobHeldMaxPreemptRequeue"], + f"Victim after {cycle} preemptions", + ) + + atf.cancel_jobs([victim]) + + +def test_preempt_does_not_trip_node_fail_requeue_limit(nodes): + """Preemption requeues must NOT count toward MaxNodeFailRequeue.""" + target_node = nodes[0] + victim = None + + try: + # Keep preemption unlimited and make the node-failure limit tiny. If + # preemption incorrectly consumed the node-failure counter, the victim + # would be held by the second preemption. + atf.set_config_parameter("MaxNodeFailRequeue", "1") + atf.set_config_parameter("MaxPreemptRequeue", "0") + atf.run_command( + "scontrol reconfigure", + user=atf.properties["slurm-user"], + fatal=True, + ) + + victim = atf.submit_job_sbatch( + f"-N1 -w {target_node} --exclusive --requeue " + f"--qos={QOS_LOW} --wrap 'sleep 600'", + fatal=True, + ) + + for cycle in range(1, 3): + atf.wait_for_job_state(victim, "RUNNING", fatal=True, timeout=60) + preemptor = atf.submit_job_sbatch( + f"-N1 -w {target_node} --exclusive " + f"--qos={QOS_HIGH} --wrap 'sleep 2'", + fatal=True, + ) + atf.repeat_until( + lambda: atf.get_job_parameter(victim, "JobState"), + lambda s: s != "RUNNING", + timeout=30, + fatal=True, + ) + atf.wait_for_job_state(preemptor, "DONE", fatal=True, timeout=30) + + reason = atf.get_job_parameter(victim, "Reason") or "" + job_state = atf.get_job_parameter(victim, "JobState") + assert "MaxNodeFailRequeue" not in reason, ( + f"Preemption should not trigger MaxNodeFailRequeue after " + f"{cycle} preemptions, Reason={reason}" + ) + assert "node_failure_requeue_limit" not in reason.lower(), ( + f"Preemption should not trigger node-failure hold after " + f"{cycle} preemptions, Reason={reason}" + ) + assert job_state != "REQUEUE_HOLD", ( + f"Victim should not be held by node-failure limit after " + f"{cycle} preemptions, state={job_state}, Reason={reason}" + ) + finally: + if victim: + atf.cancel_jobs([victim]) + + atf.set_config_parameter("MaxNodeFailRequeue", str(NODE_FAIL_LIMIT)) + atf.set_config_parameter("MaxPreemptRequeue", str(PREEMPT_LIMIT)) + atf.run_command( + "scontrol reconfigure", + user=atf.properties["slurm-user"], + fatal=True, + ) + + +def test_requeuehold_exempt_from_counting(): + """scontrol requeuehold must NOT count toward MaxPreemptRequeue.""" + job_id = atf.submit_job_sbatch( + "-N1 --exclusive --requeue --wrap 'sleep 600'", + fatal=True, + ) + try: + for cycle in range(1, PREEMPT_LIMIT + 2): + atf.wait_for_job_state(job_id, "RUNNING", fatal=True, timeout=60) + + # requeuehold - should NOT count + atf.run_command( + f"scontrol requeuehold {job_id}", + user=atf.properties["slurm-user"], + fatal=True, + ) + atf.repeat_until( + lambda: ( + atf.get_job_parameter(job_id, "Reason") or "", + atf.get_job_parameter(job_id, "Priority"), + atf.get_job_parameter(job_id, "JobState"), + ), + lambda result: ( + "job_requeued_in_held_state" in result[0].lower() + or "JobHeldUser" in result[0] + or "JobHeldAdmin" in result[0] + ) + and str(result[1]) == "0" + and result[2] in ("PENDING", "REQUEUE_HOLD"), + timeout=30, + fatal=True, + ) + job_state = atf.get_job_parameter(job_id, "JobState") + reason = atf.get_job_parameter(job_id, "Reason") or "" + priority = atf.get_job_parameter(job_id, "Priority") + assert job_state in ("PENDING", "REQUEUE_HOLD"), ( + f"requeuehold should leave job held pending, got {job_state}" + ) + assert ( + "job_requeued_in_held_state" in reason.lower() + or "JobHeldUser" in reason + or "JobHeldAdmin" in reason + ), ( + f"requeuehold should keep a normal hold reason after " + f"{cycle} attempts, got {reason}" + ) + assert "MaxPreemptRequeue" not in reason, ( + f"requeuehold should NOT trigger preemption limit after " + f"{cycle} attempts, Reason={reason}" + ) + assert "preemption_requeue_limit" not in reason.lower(), ( + f"requeuehold should NOT relabel the hold as preemption " + f"after {cycle} attempts, Reason={reason}" + ) + assert str(priority) == "0", ( + f"requeuehold job should have Priority=0, got {priority}" + ) + + if cycle <= PREEMPT_LIMIT: + atf.run_command( + f"scontrol release {job_id}", + user=atf.properties["slurm-user"], + fatal=True, + ) + finally: + atf.cancel_jobs([job_id]) + + +def test_user_requeue_exempt_from_counting(): + """User-initiated scontrol requeue must NOT count toward MaxPreemptRequeue.""" + job_id = atf.submit_job_sbatch( + "-N1 --exclusive --requeue --wrap 'sleep 600'", + fatal=True, + ) + + try: + for cycle in range(PREEMPT_LIMIT + 2): + atf.wait_for_job_state(job_id, "RUNNING", fatal=True, timeout=60) + atf.run_command( + f"scontrol requeue {job_id}", + user=atf.properties["test-user"], + fatal=True, + ) + reason = atf.get_job_parameter(job_id, "Reason") or "" + assert ( + "MaxPreemptRequeue" not in reason + and "preemption_requeue_limit" not in reason.lower() + ), ( + f"user requeue should not trigger preemption limit after " + f"{cycle + 1} requeues, Reason={reason}" + ) + finally: + atf.cancel_jobs([job_id]) + + +def test_operator_requeue_counts_toward_preempt_limit(): + """Operator-initiated scontrol requeue counts toward MaxPreemptRequeue.""" + job_id = atf.submit_job_sbatch( + "-N1 --exclusive --requeue --wrap 'sleep 600'", + fatal=True, + ) + + try: + for cycle in range(1, PREEMPT_LIMIT + 2): + atf.wait_for_job_state(job_id, "RUNNING", fatal=True, timeout=60) + atf.run_command( + f"scontrol requeue {job_id}", + user=atf.properties["slurm-user"], + fatal=True, + ) + reason = atf.get_job_parameter(job_id, "Reason") or "" + + if cycle <= PREEMPT_LIMIT: + assert "MaxPreemptRequeue" not in reason, ( + f"operator requeue should not hold before limit " + f"{PREEMPT_LIMIT}, cycle={cycle}, Reason={reason}" + ) + else: + atf.wait_for_job_state( + job_id, "REQUEUE_HOLD", fatal=True, timeout=30 + ) + _assert_requeue_hold( + job_id, + [ + "preemption_requeue_limit", + "JobHeldMaxPreemptRequeue", + ], + f"operator requeue after {cycle} requeues", + ) + finally: + atf.cancel_jobs([job_id]) + + +def test_release_resets_counters(nodes): + """scontrol release must reset per-cause counters.""" + target_node, other_node = nodes + + filler = atf.submit_job_sbatch( + f"-N1 -w {other_node} --exclusive --requeue --wrap 'sleep 600'", + fatal=True, + ) + atf.wait_for_job_state(filler, "RUNNING", fatal=True) + + job_id = atf.submit_job_sbatch( + f"-N1 -w {target_node} --exclusive --requeue --wrap 'sleep 600'", + fatal=True, + ) + atf.wait_for_job_state(job_id, "RUNNING", fatal=True) + + # Hit the node-fail limit + for cycle in range(NODE_FAIL_LIMIT + 1): + atf.run_command( + f"scontrol update nodename={target_node} state=DOWN reason=test_{cycle}", + user=atf.properties["slurm-user"], + fatal=True, + ) + atf.repeat_until( + lambda: atf.get_job_parameter(job_id, "JobState"), + lambda s: s != "RUNNING", + timeout=30, + fatal=True, + ) + if cycle < NODE_FAIL_LIMIT: + atf.run_command( + f"scontrol update nodename={target_node} state=RESUME", + user=atf.properties["slurm-user"], + fatal=True, + ) + atf.wait_for_node_state(target_node, "IDLE", fatal=True) + atf.wait_for_job_state(job_id, "RUNNING", fatal=True) + + # Job should be held now + _assert_requeue_hold( + job_id, + ["node_failure_requeue_limit", "JobHeldMaxNodeFailRequeue"], + "job at node-failure limit before release", + ) + + # Release the hold - counters should reset + atf.run_command( + f"scontrol update nodename={target_node} state=RESUME", + user=atf.properties["slurm-user"], + fatal=True, + ) + atf.wait_for_node_state(target_node, "IDLE", fatal=True) + atf.run_command( + f"scontrol release {job_id}", + user=atf.properties["slurm-user"], + fatal=True, + ) + atf.wait_for_job_state(job_id, "RUNNING", fatal=True, timeout=60) + + # One more node failure should NOT immediately re-hold + # (counter was reset by release) + atf.run_command( + f"scontrol update nodename={target_node} state=DOWN reason=after_release", + user=atf.properties["slurm-user"], + fatal=True, + ) + atf.repeat_until( + lambda: atf.get_job_parameter(job_id, "JobState"), + lambda s: s != "RUNNING", + timeout=30, + fatal=True, + ) + reason = atf.get_job_parameter(job_id, "Reason") or "" + assert "MaxNodeFail" not in reason, ( + f"After release, one node failure should NOT re-hold " + f"(counters should be reset), Reason={reason}" + ) + + atf.cancel_jobs([filler, job_id]) + + +def test_unlimited_preempt_requeue(nodes): + """MaxPreemptRequeue=0 (the default) means unlimited - never held for preemption.""" + target_node = nodes[0] + victim = None + + try: + # 0 is the built-in default for MaxPreemptRequeue (see read_config.h); + # set it explicitly because the module fixture pins a positive limit. + atf.set_config_parameter("MaxPreemptRequeue", "0") + atf.run_command( + "scontrol reconfigure", + user=atf.properties["slurm-user"], + fatal=True, + ) + assert str(atf.get_config_parameter("MaxPreemptRequeue")) == "0", ( + "MaxPreemptRequeue should read 0 (unlimited) after being configured" + ) + + victim = atf.submit_job_sbatch( + f"-N1 -w {target_node} --exclusive --requeue " + f"--qos={QOS_LOW} --wrap 'sleep 600'", + fatal=True, + ) + + # Preempt more times than the original limit + for cycle in range(PREEMPT_LIMIT + 2): + atf.wait_for_job_state(victim, "RUNNING", fatal=True, timeout=60) + preemptor = atf.submit_job_sbatch( + f"-N1 -w {target_node} --exclusive " + f"--qos={QOS_HIGH} --wrap 'sleep 2'", + fatal=True, + ) + atf.wait_for_job_state(preemptor, "DONE", fatal=True, timeout=30) + + # Victim should still be PENDING (not held) + reason = atf.get_job_parameter(victim, "Reason") or "" + priority = atf.get_job_parameter(victim, "Priority") + assert "MaxPreemptRequeue" not in reason, ( + f"With MaxPreemptRequeue=0, job should never be held for " + f"preemption, got Reason={reason}" + ) + assert str(priority) != "0", ( + f"Job should not have Priority=0 with unlimited preemption" + ) + finally: + if victim: + atf.cancel_jobs([victim]) + + # Restore original limit + atf.set_config_parameter("MaxPreemptRequeue", str(PREEMPT_LIMIT)) + atf.run_command( + "scontrol reconfigure", + user=atf.properties["slurm-user"], + fatal=True, + )