testsuite/python/tests/test_162_3.py

############################################################################
# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
############################################################################
"""Stepmgr/async-step integration tests.

These tests pin operational properties of async steps that go beyond the
srun CLI surface (covered in test_116_55.py): scontrol visibility of the
running step, and stepmgr state persistence across `scontrol reconfigure`.
"""

import re

import pytest

import atf

PARSABLE_RE = re.compile(r"^(\d+)\.(\d+)$", re.MULTILINE)


@pytest.fixture(scope="module", autouse=True)
def setup():
    atf.require_version(
        (26, 5),
        "bin/srun",
        reason="Issue 50739: srun --async was added in 26.05",
    )
    atf.require_config_parameter_includes("SlurmctldParameters", "enable_stepmgr")
    atf.require_config_parameter_includes("PrologFlags", "Contain")
    atf.require_accounting()
    atf.require_nodes(1, [("CPUs", 2)])
    atf.require_slurm_running()


def test_async_step_state_visible_in_scontrol():
    """A running async step is reported by `scontrol show steps` with State=RUNNING.

    Confirms stepmgr maintains the step record after srun has exited and
    that the basic JobId/StepId fields match the values srun reported.
    """
    out_file = "state.out"

    job_id = atf.submit_job_sbatch(
        f"-N1 -n1 -t1 --wrap 'srun --async -o {out_file} sleep 30; sleep 30'",
        fatal=True,
    )
    step_key = f"{job_id}.0"

    state = None
    for _t in atf.timer(quiet=True):
        steps = atf.get_steps(job_id, quiet=True)
        if step_key in steps:
            state = steps[step_key].get("State")
            if state == "RUNNING":
                break
    else:
        pytest.fail(
            f"Async step {step_key} did not reach RUNNING in scontrol; "
            f"last observed State={state!r}"
        )

    step = steps[step_key]
    assert step["StepId"] == step_key, f"Step StepId should be {step_key}: {step!r}"

    atf.run_command(f"scancel {job_id}", quiet=True)
    atf.wait_for_job_state(job_id, "DONE")


def test_async_step_survives_reconfigure():
    """A queued async step still launches after `scontrol reconfigure`.

    Stepmgr state (the pending async step held in the job's step list)
    must be preserved across a controller reconfigure.
    """
    out1 = "rcfg1.out"
    out2 = "rcfg2.out"

    marker = "rcfg_marker"

    script = "rcfg_submit.sh"
    release = "rcfg_release"
    # Step 1 holds the exclusive resource until the release file is written,
    # giving deterministic control over when step 2 can promote from PENDING.
    atf.make_bash_script(
        script,
        f"""srun --async --parsable --exclusive -o {out1} \\
    bash -c 'until [ -f {release} ]; do sleep 0.2; done' > {marker} 2>&1
srun --async --parsable --exclusive -o {out2} bash -c 'echo second-ran' >> {marker} 2>&1
swait
""",
    )
    job_id = atf.submit_job_sbatch(f"-N1 -n1 -t2 {script}", fatal=True)
    atf.wait_for_file(marker, fatal=True)

    text = ""
    for _t in atf.timer():
        text = atf.run_command_output(f"cat {marker}", quiet=True)
        if len(PARSABLE_RE.findall(text)) >= 2:
            break
    else:
        pytest.fail(f"Expected two parsable step-id lines, got: {text!r}")

    matches = PARSABLE_RE.findall(text)
    second_step = int(matches[1][1])
    second_key = f"{job_id}.{second_step}"

    saw_pending = False
    # Poll fast: the queued step may be PENDING only briefly.
    for _t in atf.timer(poll_interval=0.5, quiet=True):
        steps = atf.get_steps(job_id, quiet=True)
        if steps.get(second_key, {}).get("State") == "PENDING":
            saw_pending = True
            break
    assert saw_pending, (
        f"Second async step {second_key} did not reach PENDING before "
        f"reconfigure; cannot exercise the survives-reconfigure path"
    )

    atf.run_command(
        "scontrol reconfigure",
        user=atf.properties["slurm-user"],
        fatal=True,
    )

    steps_after = atf.get_steps(job_id, quiet=True)
    state_after = steps_after.get(second_key, {}).get("State")
    assert state_after == "PENDING", (
        f"Second async step {second_key} should still be PENDING "
        f"immediately after scontrol reconfigure; got State={state_after!r}"
    )

    atf.run_command(f"touch {release}", fatal=True)

    assert atf.wait_for_step_accounted(
        job_id, second_step, fatal=True
    ), f"async step {job_id}.{second_step} should be recorded in accounting"
    atf.wait_for_job_state(job_id, "DONE", fatal=True)
    atf.wait_for_file(out2, fatal=True)
    out2_text = atf.run_command_output(f"cat {out2}", fatal=True)
    assert (
        "second-ran" in out2_text
    ), f"Queued async step should run after reconfigure; got {out2_text!r}"