| ############################################################################ | |
| # Copyright (C) SchedMD LLC. | |
| ############################################################################ | |
| import time | |
| import pytest | |
| import atf | |
| pytestmark = pytest.mark.slow | |
| node_prefix = "node" | |
| # Note that power_interval has to be at least 5 seconds, recommended 10 seconds | |
| # for later tests | |
| power_interval = 10 | |
| suspend_time = 10 | |
| suspend_timeout = 10 | |
| resume_timeout = 10 | |
| @pytest.fixture(scope="module", autouse=True) | |
| def setup(): | |
| atf.require_auto_config("Runs slurmd on same machine as slurmctld") | |
| atf.require_config_parameter("NodeFeaturesPlugins", "node_features/helpers") | |
| atf.require_config_parameter("TreeWidth", 65533) | |
| atf.require_config_parameter("ResumeProgram", "/bin/true") | |
| atf.require_config_parameter("SuspendProgram", "/bin/true") | |
| # Time for cloud node to sit idle with no jobs before told to power down | |
| atf.require_config_parameter("SuspendTime", suspend_time) | |
| # Time allowed for cloud node to finish POWERING_DOWN | |
| atf.require_config_parameter("SuspendTimeout", suspend_timeout) | |
| # Time to wait for cloud node to power up and register with slurmctld after | |
| # being ALLOCATED job | |
| atf.require_config_parameter("ResumeTimeout", resume_timeout) | |
| # Mark nodes as IDLE, regardless of current state, when suspending nodes with | |
| # SuspendProgram so that nodes will be eligible to be resumed at a later time | |
| atf.require_config_parameter_includes("SlurmctldParameters", "idle_on_node_suspend") | |
| # Register the cloud node in slurm.conf | |
| atf.require_config_parameter( | |
| "NodeName", {f"{node_prefix}1": {"Feature": "f1,nf1", "State": "CLOUD"}} | |
| ) | |
| # Create set/group of nodes that have feature f1 and assign to partitions | |
| atf.require_config_parameter("Nodeset", {"ns1": {"Feature": "f1"}}) | |
| atf.require_config_parameter( | |
| "PartitionName", | |
| { | |
| "primary": {"Nodes": "ALL"}, | |
| "cloud1": {"Nodes": "ns1"}, | |
| "powerdownonidle": {"Nodes": "ns1", "PowerDownOnIdle": "yes"}, | |
| }, | |
| ) | |
| # Define nf1 feature in 'helpers.conf'. Used for testing Node Features | |
| atf.require_config_parameter( | |
| "NodeName", | |
| {f"{node_prefix}1": {"Feature": "nf1", "Helper": "/bin/true"}}, | |
| source="helpers", | |
| ) | |
| # Don't run the usual atf.require_slurm_running() because tests will start | |
| # slurmds manually | |
| atf.start_slurmctld(clean=True) | |
| yield | |
| # Have to manually kill the slurmctld and slurmd in the teardown | |
| atf.stop_slurmctld(also_slurmds=True) | |
| @pytest.fixture(scope="function", autouse=True) | |
| def cloud_state(): | |
| """stop all jobs and reset cloud node state after each test""" | |
| yield | |
| atf.cancel_all_jobs() | |
| atf.stop_slurmd() | |
| atf.restart_slurmctld(clean=True) | |
| time.sleep(2) | |
| # Tests | |
| # Test state cycle of cloud nodes: POWERED_DOWN, POWERING_UP, IDLE, | |
| # POWERING_DOWN, POWERED_DOWN | |
| def test_cloud_state_cycle(): | |
| assert "CLOUD" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node state should always contain CLOUD" | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node must be in IDLE state" | |
| assert "POWERED_DOWN" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node must be in POWERED_DOWN state" | |
| # Schedule a job to cloud node's partition, transitioning node to ALLOCATED | |
| # and POWERING_UP state | |
| job_id = atf.submit_job_sbatch("-p cloud1 --wrap 'srun hostname'", fatal=True) | |
| atf.wait_for_node_state(f"{node_prefix}1", "ALLOCATED", timeout=5, fatal=True) | |
| atf.wait_for_node_state(f"{node_prefix}1", "POWERING_UP", fatal=True) | |
| assert "CONFIGURING" == atf.get_job_parameter( | |
| job_id, "JobState", default="NOT_FOUND", quiet=True | |
| ), "Submitted job should be in CONFIGURING state while its ALLOCATED cloud node is POWERING_UP" | |
| # TODO: Wait 2 seconds to avoid race condition between slurmd and slurmctld | |
| # Remove once bug 16459 is fixed. | |
| time.sleep(2) | |
| # Register the new slurmd | |
| atf.start_slurmd(f"{node_prefix}1", "-b --conf 'feature=f1'", wait=None) | |
| # Make sure the cloud node resumes | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", | |
| "POWERING_UP", | |
| reverse=True, | |
| timeout=resume_timeout + 5, | |
| fatal=True, | |
| ) | |
| # Cloud node takes PERIODIC_TIMEOUT seconds to register and resume. | |
| # The job has 45 seconds to finish | |
| atf.wait_for_job_state( | |
| job_id, "COMPLETED", timeout=atf.PERIODIC_TIMEOUT + 15, fatal=True | |
| ) | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "Per 'SlurmctldParameters=idle_on_node_suspend' in slurm.conf, cloud node should always be in IDLE state except when ALLOCATED/MIXED for an assigned job" | |
| # Make sure the cloud node starts suspending once it hasn't received a job | |
| # for suspend_time | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", | |
| ["POWER_DOWN", "POWERING_DOWN"], | |
| timeout=suspend_time + 5, | |
| fatal=True, | |
| ) | |
| # Make sure the cloud node is fully POWERED_DOWN by suspend_timeout | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", "POWERED_DOWN", timeout=suspend_timeout + 5, fatal=True | |
| ) | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "Per 'SlurmctldParameters=idle_on_node_suspend' in slurm.conf, cloud node should always be in IDLE state except when ALLOCATED/MIXED for an assigned job" | |
| # Test that cloud node powers down if exceeding resume_timeout | |
| def test_resume_timeout(): | |
| assert "CLOUD" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node state should always contain CLOUD" | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node must be in IDLE state" | |
| assert "POWERED_DOWN" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node must be in POWERED_DOWN state" | |
| # Schedule a job to cloud node's partition, transitioning node to ALLOCATED | |
| # and POWERING_UP state | |
| job_id = atf.submit_job_sbatch("-p cloud1 --wrap 'srun hostname'", fatal=True) | |
| atf.wait_for_node_state(f"{node_prefix}1", "ALLOCATED", timeout=5, fatal=True) | |
| atf.wait_for_node_state(f"{node_prefix}1", "POWERING_UP", fatal=True) | |
| assert "CONFIGURING" == atf.get_job_parameter( | |
| job_id, "JobState", default="NOT_FOUND", quiet=True | |
| ), "Submitted job should be in CONFIGURING state while its ALLOCATED cloud node is POWERING_UP" | |
| # Never spin up the accompanying slurmd, waiting resume_timeout for cloud | |
| # node to be DOWN | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", "DOWN", timeout=resume_timeout + 5, fatal=True | |
| ) | |
| # Assert surpassing resume_timeout correctly set cloud node's state | |
| assert "POWERED_DOWN" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "Cloud node didn't enter POWERED_DOWN state immediately after surpassing resume timeout" | |
| # Test scontrol setting cloud node state using POWER_UP and POWER_DOWN | |
| def test_scontrol_power_up_down(): | |
| assert "CLOUD" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node state should always contain CLOUD" | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node must be in IDLE state" | |
| assert "POWERED_DOWN" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node must be in POWERED_DOWN state" | |
| # scontrol POWER_UP | |
| atf.run_command( | |
| f"scontrol update nodename={node_prefix}1 state=POWER_UP", | |
| fatal=True, | |
| user="slurm", | |
| ) | |
| node_state = set(atf.get_node_parameter(f"{node_prefix}1", "state")) | |
| assert ( | |
| set(["POWER_UP", "POWERING_UP"]) & node_state | |
| ), "Cloud node should enter powering up process after scontrol command" | |
| atf.wait_for_node_state(f"{node_prefix}1", "POWERING_UP", fatal=True) | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "Per 'SlurmctldParameters=idle_on_node_suspend' in slurm.conf, cloud node should always be in IDLE state except when ALLOCATED/MIXED for an assigned job" | |
| # TODO: Wait 2 seconds to avoid race condition between slurmd and slurmctld | |
| # Remove once bug 16459 is fixed. | |
| time.sleep(2) | |
| # Register the new slurmd | |
| atf.start_slurmd(f"{node_prefix}1", "-b --conf 'feature=f1'", wait=None) | |
| # Make sure the cloud node resumes | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", | |
| "POWERING_UP", | |
| reverse=True, | |
| timeout=resume_timeout + 5, | |
| fatal=True, | |
| ) | |
| # scontrol POWER_DOWN | |
| atf.run_command( | |
| f"scontrol update nodename={node_prefix}1 state=POWER_DOWN", | |
| fatal=True, | |
| user="slurm", | |
| ) | |
| node_state = set(atf.get_node_parameter(f"{node_prefix}1", "state")) | |
| assert ( | |
| set(["POWER_DOWN", "POWERING_DOWN"]) & node_state | |
| ), "POWER_DOWN should immediately be added to cloud node's state" | |
| atf.wait_for_node_state(f"{node_prefix}1", "POWERING_DOWN", fatal=True) | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", "POWERED_DOWN", timeout=suspend_timeout + 5, fatal=True | |
| ) | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "Per 'SlurmctldParameters=idle_on_node_suspend' in slurm.conf, cloud node should always be in IDLE state except when ALLOCATED/MIXED for an assigned job" | |
| # Test scontrol setting cloud node state using POWER_DOWN_ASAP | |
| def test_scontrol_power_down_asap(): | |
| assert "CLOUD" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node state should always contain CLOUD" | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node must be in IDLE state" | |
| assert "POWERED_DOWN" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node must be in POWERED_DOWN state" | |
| # scontrol POWER_UP | |
| atf.run_command( | |
| f"scontrol update nodename={node_prefix}1 state=POWER_UP", | |
| fatal=True, | |
| user="slurm", | |
| ) | |
| node_state = set(atf.get_node_parameter(f"{node_prefix}1", "state")) | |
| assert ( | |
| set(["POWER_UP", "POWERING_UP"]) & node_state | |
| ), "Cloud node should enter powering up process after scontrol command" | |
| atf.wait_for_node_state(f"{node_prefix}1", "POWERING_UP", fatal=True) | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "Per 'SlurmctldParameters=idle_on_node_suspend' in slurm.conf, cloud node should always be in IDLE state except when ALLOCATED/MIXED for an assigned job" | |
| # TODO: Wait 2 seconds to avoid race condition between slurmd and slurmctld | |
| # Remove once bug 16459 is fixed. | |
| time.sleep(2) | |
| # Register the new slurmd | |
| atf.start_slurmd(f"{node_prefix}1", "-b --conf 'feature=f1'", wait=None) | |
| # Make sure the cloud node resumes | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", | |
| "POWERING_UP", | |
| reverse=True, | |
| timeout=resume_timeout + 5, | |
| fatal=True, | |
| ) | |
| # Submit job in preparation for POWER_DOWN_ASAP | |
| job_id = atf.submit_job_sbatch("-p cloud1 --wrap 'srun sleep 5'", fatal=True) | |
| atf.wait_for_job_state( | |
| job_id, | |
| "RUNNING", | |
| timeout=atf.PERIODIC_TIMEOUT + atf.default_command_timeout, | |
| fatal=True, | |
| ) | |
| assert "ALLOCATED" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "Cloud node should be ALLOCATED while running a job" | |
| # scontrol POWER_DOWN_ASAP | |
| atf.run_command( | |
| f"scontrol update nodename={node_prefix}1 state=POWER_DOWN_ASAP", | |
| fatal=True, | |
| user="slurm", | |
| ) | |
| assert "POWER_DOWN" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "POWER_DOWN should immediately be added to cloud node's state" | |
| assert "DRAIN" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "Cloud node should be in DRAIN state to prepare for POWER_DOWN state" | |
| # Wait for job to finish and then assert cloud node becomes POWERED_DOWN | |
| atf.wait_for_job_state(job_id, "COMPLETED", fatal=True) | |
| atf.wait_for_node_state(f"{node_prefix}1", "POWERING_DOWN", fatal=True) | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", "POWERED_DOWN", timeout=suspend_timeout + 5, fatal=True | |
| ) | |
| # Test scontrol setting cloud node state using POWER_DOWN_FORCE and RESUME | |
| def test_scontrol_power_down_force_and_resume(): | |
| assert "CLOUD" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node state should always contain CLOUD" | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node must be in IDLE state" | |
| assert "POWERED_DOWN" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node must be in POWERED_DOWN state" | |
| # scontrol POWER_UP | |
| atf.run_command( | |
| f"scontrol update nodename={node_prefix}1 state=POWER_UP", | |
| fatal=True, | |
| user="slurm", | |
| ) | |
| node_state = set(atf.get_node_parameter(f"{node_prefix}1", "state")) | |
| assert ( | |
| set(["POWER_UP", "POWERING_UP"]) & node_state | |
| ), "Cloud node should enter powering up process after scontrol command" | |
| atf.wait_for_node_state(f"{node_prefix}1", "POWERING_UP", fatal=True) | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "Per 'SlurmctldParameters=idle_on_node_suspend' in slurm.conf, cloud node should always be in IDLE state except when ALLOCATED/MIXED for an assigned job" | |
| # TODO: Wait 2 seconds to avoid race condition between slurmd and slurmctld | |
| # Remove once bug 16459 is fixed. | |
| time.sleep(2) | |
| # Register the new slurmd | |
| atf.start_slurmd(f"{node_prefix}1", "-b --conf 'feature=f1'", wait=None) | |
| # Make sure the cloud node resumes | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", | |
| "POWERING_UP", | |
| reverse=True, | |
| timeout=resume_timeout + 5, | |
| fatal=True, | |
| ) | |
| # Submit job in preparation for POWER_DOWN_FORCE to cancel | |
| job_id = atf.submit_job_sbatch("-p cloud1 --wrap 'srun sleep 300'", fatal=True) | |
| atf.wait_for_job_state( | |
| job_id, "RUNNING", timeout=atf.PERIODIC_TIMEOUT + 5, fatal=True | |
| ) | |
| assert "ALLOCATED" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "Cloud node should be ALLOCATED while running a job" | |
| # scontrol POWER_DOWN_FORCE when cloud node is already up | |
| atf.run_command( | |
| f"scontrol update nodename={node_prefix}1 state=POWER_DOWN_FORCE", | |
| fatal=True, | |
| user="slurm", | |
| ) | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "Per 'SlurmctldParameters=idle_on_node_suspend' in slurm.conf, cloud node should always be in IDLE state except when ALLOCATED/MIXED for an assigned job" | |
| # Make sure job is requeued and cloud node is POWERED_DOWN | |
| node_state = set(atf.get_node_parameter(f"{node_prefix}1", "state")) | |
| assert ( | |
| set(["POWER_DOWN", "POWERING_DOWN"]) & node_state | |
| ), "Cloud node should enter powering down process immediately after scontrol command" | |
| atf.wait_for_node_state(f"{node_prefix}1", "POWERING_DOWN", fatal=True) | |
| assert atf.wait_for_job_state( | |
| job_id, "PENDING", timeout=10, fatal=True | |
| ), "Job should be requeued and PENDING after node is powered down" | |
| # Test scontrol RESUME sets cloud node to POWERED_DOWN when POWERING_DOWN | |
| # already | |
| atf.run_command( | |
| f"scontrol update nodename={node_prefix}1 state=RESUME", | |
| fatal=True, | |
| user="slurm", | |
| ) | |
| atf.wait_for_node_state(f"{node_prefix}1", "POWERED_DOWN", timeout=5, fatal=True) | |
| # Test cloud nodes POWER_DOWN and then power up with different ActiveFeatures | |
| # to handle jobs | |
| def test_node_features(): | |
| assert "CLOUD" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node state should always contain CLOUD" | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node must be in IDLE state" | |
| assert "POWERED_DOWN" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node must be in POWERED_DOWN state" | |
| # Periodically make sure that the AvailableFeatures for the cloud node are correct | |
| available_features = sorted(["f1", "nf1"]) | |
| assert available_features == sorted( | |
| atf.get_node_parameter(f"{node_prefix}1", "features") | |
| ), "Cloud node's AvailableFeatures don't match what was set in slurm.conf" | |
| # Schedule a job to cloud node's partition, transitioning node to ALLOCATED | |
| # and POWERING_UP state | |
| job_id = atf.submit_job_sbatch("-p cloud1 -C f1 --wrap 'srun hostname'", fatal=True) | |
| atf.wait_for_node_state(f"{node_prefix}1", "ALLOCATED", timeout=5, fatal=True) | |
| atf.wait_for_node_state(f"{node_prefix}1", "POWERING_UP", fatal=True) | |
| assert "CONFIGURING" == atf.get_job_parameter( | |
| job_id, "JobState", default="NOT_FOUND", quiet=True | |
| ), "Submitted job should be in CONFIGURING state while its ALLOCATED cloud node is POWERING_UP" | |
| # TODO: Wait 2 seconds to avoid race condition between slurmd and slurmctld | |
| # Remove once bug 16459 is fixed. | |
| time.sleep(2) | |
| # Register the new slurmd | |
| atf.start_slurmd(f"{node_prefix}1", "-b --conf 'feature=f1'", wait=None) | |
| # Make sure the cloud node resumes | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", | |
| "POWERING_UP", | |
| reverse=True, | |
| timeout=resume_timeout + 5, | |
| fatal=True, | |
| ) | |
| # Assert cloud node maintains expected configuration of features (not nf1) | |
| assert available_features == sorted( | |
| atf.get_node_parameter(f"{node_prefix}1", "features") | |
| ), "Cloud node's AvailableFeatures don't match what was set in slurm.conf" | |
| assert ["f1"] == atf.get_node_parameter( | |
| f"{node_prefix}1", "active_features" | |
| ), "Cloud node should only have the 'f1' feature when none are explicitly requested" | |
| # Cloud node takes PERIODIC_TIMEOUT seconds to register and resume. | |
| # The job has 45 seconds to finish | |
| atf.wait_for_job_state( | |
| job_id, "COMPLETED", timeout=atf.PERIODIC_TIMEOUT + 15, fatal=True | |
| ) | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "Per 'SlurmctldParameters=idle_on_node_suspend' in slurm.conf, cloud node should always be in IDLE state except when ALLOCATED/MIXED for an assigned job" | |
| # Now submit job requiring nf1 feature, which cloud node doesn't currently | |
| # have active | |
| job_id = atf.submit_job_sbatch( | |
| f"-p cloud1 -C nf1 -w {node_prefix}1 --wrap 'srun hostname'", fatal=True | |
| ) | |
| # Make sure cloud node enters POWERING_DOWN state without running the job | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", "POWERING_DOWN", timeout=suspend_time + 15, fatal=True | |
| ) | |
| assert "PENDING" == atf.get_job_parameter( | |
| job_id, "JobState", default="NOT_FOUND", quiet=True | |
| ), "Job should be pending because there are no cloud nodes with the requested feature active" | |
| # Cloud node should enter "POWERED_DOWN" state | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", "POWERED_DOWN", timeout=suspend_timeout + 5, fatal=True | |
| ) | |
| # Should become ALLOCATED once scheduler runs, notwithstanding whether node | |
| # was already PLANNED or not | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", "ALLOCATED", timeout=atf.PERIODIC_TIMEOUT + 15, fatal=True | |
| ) | |
| # Cloud node then enters POWERING_UP once the power thread runs | |
| atf.wait_for_node_state(f"{node_prefix}1", "POWERING_UP", fatal=True) | |
| assert "CONFIGURING" == atf.get_job_parameter( | |
| job_id, "JobState", default="NOT_FOUND", quiet=True | |
| ), "Submitted job should be in CONFIGURING state while its ALLOCATED cloud node is POWERING_UP" | |
| # TODO: Wait 2 seconds to avoid race condition between slurmd and slurmctld | |
| # Remove once bug 16459 is fixed. | |
| time.sleep(2) | |
| # Register the new slurmd | |
| atf.start_slurmd(f"{node_prefix}1", "-b --conf 'feature=f1'", wait=None) | |
| # Make sure the cloud node resumes | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", | |
| "POWERING_UP", | |
| reverse=True, | |
| timeout=resume_timeout + 5, | |
| fatal=True, | |
| ) | |
| # Test the cloud node now has the activated feature needed to run the job | |
| assert available_features == sorted( | |
| atf.get_node_parameter(f"{node_prefix}1", "features") | |
| ), "Cloud node's AvailableFeatures don't match what was set in slurm.conf" | |
| assert available_features == sorted( | |
| atf.get_node_parameter(f"{node_prefix}1", "active_features") | |
| ), "Cloud node should have both of its available features active" | |
| # Cloud node takes PERIODIC_TIMEOUT seconds to register and resume. | |
| # The job has 45 seconds to finish | |
| atf.wait_for_job_state( | |
| job_id, "COMPLETED", timeout=atf.PERIODIC_TIMEOUT + 15, fatal=True | |
| ) | |
| atf.wait_for_node_state(f"{node_prefix}1", "IDLE", timeout=5, fatal=True) | |
| # Test partition flag 'PowerDownOnIdle=yes' | |
| def test_power_down_on_idle(): | |
| assert "CLOUD" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node state should always contain CLOUD" | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node must be in IDLE state" | |
| assert "POWERED_DOWN" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node must be in POWERED_DOWN state" | |
| # Schedule a job to cloud node's partition, transitioning node to ALLOCATED | |
| # and POWERING_UP state | |
| job_id = atf.submit_job_sbatch( | |
| "-p powerdownonidle --wrap 'srun hostname'", fatal=True | |
| ) | |
| atf.wait_for_node_state(f"{node_prefix}1", "ALLOCATED", timeout=5, fatal=True) | |
| atf.wait_for_node_state(f"{node_prefix}1", "POWERING_UP", fatal=True) | |
| assert "CONFIGURING" == atf.get_job_parameter( | |
| job_id, "JobState", default="NOT_FOUND", quiet=True | |
| ), "Submitted job should be in CONFIGURING state while its ALLOCATED cloud node is POWERING_UP" | |
| # TODO: Wait 2 seconds to avoid race condition between slurmd and slurmctld | |
| # Remove once bug 16459 is fixed. | |
| time.sleep(2) | |
| # Register the new slurmd | |
| atf.start_slurmd(f"{node_prefix}1", "-b --conf 'feature=f1'", wait=None) | |
| # Make sure the cloud node resumes | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", | |
| "POWERING_UP", | |
| reverse=True, | |
| timeout=resume_timeout + 5, | |
| fatal=True, | |
| ) | |
| # Cloud node takes PERIODIC_TIMEOUT seconds to register and resume. | |
| # The job has 45 seconds to finish | |
| atf.wait_for_job_state( | |
| job_id, "COMPLETED", timeout=atf.PERIODIC_TIMEOUT + 15, fatal=True | |
| ) | |
| # Immediately upon job completion and becoming IDLE, cloud node should | |
| # POWER_DOWN | |
| node_state = set(atf.get_node_parameter(f"{node_prefix}1", "state")) | |
| assert ( | |
| set(["POWER_DOWN", "POWERING_DOWN"]) & node_state | |
| ), "Cloud node wasn't immediately POWER_DOWN once idle, in contrary to 'PowerDownOnIdle=yes' flag for partition" | |
| atf.wait_for_node_state(f"{node_prefix}1", "POWERING_DOWN", fatal=True) | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", "POWERED_DOWN", timeout=suspend_timeout + 5, fatal=True | |
| ) | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "Per 'SlurmctldParameters=idle_on_node_suspend' in slurm.conf, cloud node should always be in IDLE state except when ALLOCATED/MIXED for an assigned job" | |
| # Test that a per-node SuspendTime overrides the global SuspendTime. The | |
| # global SuspendTime is 10s (see the module fixture); set a longer per-node | |
| # value so the power-save scan at the global interval must NOT suspend the | |
| # node, proving the node-level value is the one in effect. | |
| @pytest.mark.skipif( | |
| atf.get_version("bin/scontrol") < (26, 11), | |
| reason="#50873: per-node SuspendTime updates added in 26.11", | |
| ) | |
| def test_per_node_suspend_time_overrides_global(): | |
| # Much longer than the global suspend_time (10) so the negative window | |
| # below has a wide, load-insensitive margin before the per-node deadline. | |
| node_suspend_time = 60 | |
| assert "POWERED_DOWN" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "Cloud node must be in POWERED_DOWN state" | |
| # Set a per-node SuspendTime distinct from the global SuspendTime. | |
| atf.run_command( | |
| f"scontrol update nodename={node_prefix}1" f" SuspendTime={node_suspend_time}", | |
| fatal=True, | |
| user=atf.properties["slurm-user"], | |
| ) | |
| assert ( | |
| atf.get_node_parameter(f"{node_prefix}1", "suspend_time")["number"] | |
| == node_suspend_time | |
| ), "Per-node SuspendTime update was not applied" | |
| # Power the node up and run a job so it becomes IDLE and the idle clock | |
| # starts, then verify the suspend happens on the per-node schedule. | |
| job_id = atf.submit_job_sbatch("-p cloud1 --wrap 'srun hostname'", fatal=True) | |
| atf.wait_for_node_state(f"{node_prefix}1", "ALLOCATED", fatal=True) | |
| atf.wait_for_node_state(f"{node_prefix}1", "POWERING_UP", fatal=True) | |
| # TODO: Wait 2 seconds to avoid race condition between slurmd and slurmctld | |
| # Remove once bug 16459 is fixed. | |
| time.sleep(2) | |
| # Register the new slurmd | |
| atf.start_slurmd(f"{node_prefix}1", "-b --conf 'feature=f1'", wait=None) | |
| # Make sure the cloud node resumes | |
| atf.wait_for_node_state(f"{node_prefix}1", "POWERING_UP", reverse=True, fatal=True) | |
| # Cloud node takes PERIODIC_TIMEOUT seconds to register and resume. | |
| # The job has 45 seconds to finish | |
| atf.wait_for_job_state( | |
| job_id, "COMPLETED", timeout=atf.PERIODIC_TIMEOUT + 15, fatal=True | |
| ) | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "Per 'SlurmctldParameters=idle_on_node_suspend' in slurm.conf, cloud node should always be in IDLE state except when ALLOCATED/MIXED for an assigned job" | |
| # The node must NOT suspend on the global SuspendTime schedule. If the | |
| # per-node value were ignored, the node would suspend on the global | |
| # suspend_time (10) -- but not necessarily at t=10: _power_save_thread() | |
| # only rescans once last_node_update moves or power_save_interval elapses, | |
| # so a broken implementation surfaces anywhere up to one power_interval | |
| # later. Watch past that whole span, not just past suspend_time. | |
| negative_window = suspend_time + power_interval + 5 | |
| assert negative_window < node_suspend_time, ( | |
| "The negative window must close before the per-node SuspendTime is due," | |
| " or a correctly-behaving node trips the assertion below" | |
| ) | |
| for _ in atf.timer(timeout=negative_window, poll_interval=1, xfail=True): | |
| node_state = set(atf.get_node_parameter(f"{node_prefix}1", "state")) | |
| assert not ({"POWER_DOWN", "POWERING_DOWN", "POWERED_DOWN"} & node_state), ( | |
| "Node suspended on the global SuspendTime schedule, but the" | |
| " per-node SuspendTime should override it" | |
| ) | |
| # The node must suspend on the per-node SuspendTime schedule. | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", | |
| ["POWER_DOWN", "POWERING_DOWN"], | |
| timeout=node_suspend_time + 15, | |
| fatal=True, | |
| ) | |
| atf.wait_for_node_state(f"{node_prefix}1", "POWERED_DOWN", fatal=True) | |
| # Make the check-in interval of the power save thread longer so we have time to | |
| # POWER_DOWN_FORCE a cloud node that's already POWERED_DOWN and ALLOCATED before | |
| # it's powered up. Saved as last test due to changing the slurm.conf file | |
| def test_scontrol_power_down_force(): | |
| assert "CLOUD" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node state should always contain CLOUD" | |
| assert "IDLE" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node must be in IDLE state" | |
| assert "POWERED_DOWN" in atf.get_node_parameter( | |
| f"{node_prefix}1", "state" | |
| ), "CLOUD node must be in POWERED_DOWN state" | |
| # Change power save thread minimum interval and restart slurmctld | |
| atf.require_config_parameter_includes( | |
| "SlurmctldParameters", ("power_save_min_interval", power_interval) | |
| ) | |
| atf.restart_slurmctld(clean=True) | |
| # Submit job, get it assigned to the cloud node, and make sure everything | |
| # goes well before cloud node enters POWERING_UP state | |
| job_id = atf.submit_job_sbatch("-p cloud1 --wrap 'srun sleep 300'", fatal=True) | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", "ALLOCATED", timeout=5, poll_interval=0.1, fatal=True | |
| ) | |
| assert atf.wait_for_job_state( | |
| job_id, "CONFIGURING" | |
| ), "Submitted job should be CONFIGURING while there is a corresponding ALLOCATED cloud node" | |
| # Now POWER_DOWN_FORCE the cloud node, make sure it enters POWER_DOWN and | |
| # never POWERING_UP, and assure that the job gets requeued | |
| assert atf.wait_for_node_state( | |
| f"{node_prefix}1", "POWERED_DOWN" | |
| ), "Couldn't run POWER_DOWN_FORCE on cloud node before it left POWERED_DOWN state. Try making power_interval longer to give the test more time" | |
| atf.run_command( | |
| f"scontrol update nodename={node_prefix}1 state=POWER_DOWN_FORCE", | |
| fatal=True, | |
| user="slurm", | |
| ) | |
| assert atf.wait_for_node_state( | |
| f"{node_prefix}1", "IDLE" | |
| ), "Per 'SlurmctldParameters=idle_on_node_suspend' in slurm.conf, cloud node should always be in IDLE state except when ALLOCATED/MIXED for an assigned job" | |
| # Handle a slight race condition where the node could be POWERING_DOWN before the check | |
| # Pretty unlikely | |
| node_state = set(atf.get_node_parameter(f"{node_prefix}1", "state")) | |
| assert ( | |
| set(["POWER_DOWN", "POWERING_DOWN"]) & node_state | |
| ), "POWER_DOWN should immediately be added to cloud node's state" | |
| assert "PENDING" == atf.get_job_parameter( | |
| job_id, "JobState", default="NOT_FOUND", quiet=True | |
| ), "Job should be PENDING after being requeued" | |
| # Make sure node finishes powering down correctly | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", "POWERING_DOWN", timeout=power_interval + 5, fatal=True | |
| ) | |
| power_down_timeout = suspend_timeout + 5 | |
| atf.wait_for_node_state( | |
| f"{node_prefix}1", | |
| "POWERED_DOWN", | |
| timeout=power_down_timeout, | |
| fatal=True, | |
| ) |