blob: 543da4db533a756e27f39178aac77449cb93a617 [file] [edit]
############################################################################
# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
############################################################################
"""ResumeTimeout accepts values too large for 16 bits.
Flexible and spot cloud reservations can take days to be fulfilled, so the
global and per-partition ResumeTimeout must allow at least 7 days. Before 26.11
ResumeTimeout was 16 bits and slurm.conf rejected values above 65535.
"""
import json
import pytest
import atf
# 7 and 14 days, both more than the old limit of 65533 seconds
g_resume_timeout = 7 * 24 * 60 * 60
p1_resume_timeout = 14 * 24 * 60 * 60
# The largest finite value. NO_VAL (4294967294) and INFINITE (4294967295) are
# the two values above it.
p3_resume_timeout = 4294967293
@pytest.fixture(scope="module", autouse=True)
def setup():
atf.require_version(
(26, 11),
"sbin/slurmd",
reason="Issue 51038: ResumeTimeout above 65533 seconds added in 26.11",
)
atf.require_version(
(26, 11),
"bin/scontrol",
reason="Issue 51038: ResumeTimeout above 65533 seconds added in 26.11",
)
atf.require_config_parameter("ResumeProgram", "/bin/true")
atf.require_config_parameter("SuspendProgram", "/bin/true")
atf.require_config_parameter("ResumeTimeout", g_resume_timeout)
atf.require_config_parameter(
"PartitionName",
{
"all": {"Nodes": "ALL"},
"p1": {"Nodes": "ALL", "ResumeTimeout": p1_resume_timeout},
"p2": {"Nodes": "ALL", "ResumeTimeout": "INFINITE"},
"p3": {"Nodes": "ALL", "ResumeTimeout": p3_resume_timeout},
},
)
atf.require_slurm_running()
def test_show_config():
"""scontrol show config reports the full global ResumeTimeout."""
assert (
atf.get_config_parameter("ResumeTimeout") == f"{g_resume_timeout} sec"
), "show config should report the full global ResumeTimeout"
def test_show_partition():
"""scontrol show partition reports the full partition ResumeTimeout."""
parts = atf.get_partitions(fatal=True)
assert (
parts["all"]["ResumeTimeout"] == "GLOBAL"
), "A partition without ResumeTimeout should show GLOBAL"
assert (
parts["p1"]["ResumeTimeout"] == p1_resume_timeout
), "show partition should report the full partition ResumeTimeout"
assert (
parts["p2"]["ResumeTimeout"] == "INFINITE"
), "A partition with ResumeTimeout=INFINITE should show INFINITE"
assert (
parts["p3"]["ResumeTimeout"] == p3_resume_timeout
), "show partition should report the largest finite ResumeTimeout"
@pytest.mark.parametrize("data_parser", ["v0.0.45", "v0.0.46"])
def test_json(data_parser):
"""data_parser v0.0.45 and v0.0.46 dump the full values. v0.0.45 keeps its
published schemas, so its global ResumeTimeout is a plain number."""
conf = json.loads(
atf.run_command_output(f"scontrol --json={data_parser} show config", fatal=True)
)["slurm_conf"]
if data_parser == "v0.0.45":
assert (
conf["ResumeTimeout"] == g_resume_timeout
), f"{data_parser} should dump the full global ResumeTimeout"
else:
assert conf["ResumeTimeout"] == {
"set": True,
"infinite": False,
"number": g_resume_timeout,
}, f"{data_parser} should dump the full global ResumeTimeout"
parts = json.loads(
atf.run_command_output(
f"scontrol --json={data_parser} show partition", fatal=True
)
)["partitions"]
resume = {part["name"]: part["timeouts"]["resume"] for part in parts}
assert resume["all"] == {
"set": False,
"infinite": False,
"number": 0,
}, f"{data_parser} should dump a partition without ResumeTimeout as unset"
assert resume["p1"] == {
"set": True,
"infinite": False,
"number": p1_resume_timeout,
}, f"{data_parser} should dump the full partition ResumeTimeout"
assert resume["p2"] == {
"set": False,
"infinite": True,
"number": 0,
}, f"{data_parser} should dump ResumeTimeout=INFINITE as infinite"
assert resume["p3"] == {
"set": True,
"infinite": False,
"number": p3_resume_timeout,
}, f"{data_parser} should dump the largest finite ResumeTimeout"
@pytest.mark.parametrize(
"value, shown, dumped",
[
pytest.param(
"INFINITE",
"infinite",
{"set": False, "infinite": True, "number": 0},
id="infinite",
),
pytest.param(
p3_resume_timeout,
f"{p3_resume_timeout} sec",
{"set": True, "infinite": False, "number": p3_resume_timeout},
id="largest",
),
],
)
def test_global_extremes(value, shown, dumped):
"""The global ResumeTimeout reports INFINITE and the largest finite value
as is, and data_parser v0.0.46 dumps them with the right flags."""
atf.set_config_parameter("ResumeTimeout", value)
assert (
atf.get_config_parameter("ResumeTimeout") == shown
), f"show config should report a global ResumeTimeout of {value}"
conf = json.loads(
atf.run_command_output("scontrol --json=v0.0.46 show config", fatal=True)
)["slurm_conf"]
assert (
conf["ResumeTimeout"] == dumped
), f"v0.0.46 should dump a global ResumeTimeout of {value}"