| #!/usr/bin/env expect |
| ############################################################################ |
| # Purpose: Test of Slurm functionality |
| # to be called from test3.11 |
| # Several cases for core based reservations using nodelists |
| # Plugin select/cons_tres needed |
| # |
| ############################################################################ |
| # Copyright (C) 2013 Barcelona Supercomputing Center |
| # |
| # This file is part of Slurm, a resource management program. |
| # For details, see <https://slurm.schedmd.com/>. |
| # Please also read the included file: DISCLAIMER. |
| # |
| # Slurm is free software; you can redistribute it and/or modify it under |
| # the terms of the GNU General Public License as published by the Free |
| # Software Foundation; either version 2 of the License, or (at your option) |
| # any later version. |
| # |
| # Slurm is distributed in the hope that it will be useful, but WITHOUT ANY |
| # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
| # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more |
| # details. |
| # |
| # You should have received a copy of the GNU General Public License along |
| # with Slurm; if not, write to the Free Software Foundation, Inc., |
| # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
| ############################################################################ |
| |
| proc inc3_11_9 {} { |
| global user_name node_count |
| global file_in bin_sleep sbatch number scontrol |
| global re_word_str scancel |
| global cluster_cpus cores_per_node def_partition |
| global def_node_name def_node_inx_min def_node_inx_max |
| global def_node_inx_min_int def_node_inx_max_int |
| |
| set res_name "resv3.11.9" |
| set res_name_test "resv3.11.9.0" |
| |
| log_info "+++++ STARTING TEST 9 +++++" |
| |
| # Assumes nodes have sequential suffix numbers |
| set min_node_inx_int $def_node_inx_min_int |
| set max_node_inx_int [expr min($def_node_inx_min_int + 4,$def_node_inx_max_int)] |
| set min_node_inx $def_node_inx_min |
| set max_node_inx $max_node_inx_int |
| set min_len [string length $min_node_inx] |
| set max_len [string length $max_node_inx] |
| if {$min_len > $max_len} { |
| # Preserve leading zeros |
| # (e.g. "nid[00001-00005]" rather than "nid[00001-5]") |
| set max_node_inx [format %${min_len}.${min_len}d $max_node_inx_int] |
| } |
| |
| # Make the job script |
| make_bash_script $file_in "$bin_sleep 100" |
| |
| # Make a reservation, just to get node size information |
| set ret_code [create_res $res_name "StartTime=now Duration=1 NodeCnt=1 User=$user_name"] |
| if {$ret_code != 0} { |
| fail "Unable to create a valid reservation" |
| } |
| |
| # Get reservation info |
| set res_info [get_reservations $res_name] |
| if { ![dict exists $res_info $res_name] } { |
| delete_res $res_name |
| fail "Unable to get info about reservation ($res_name)" |
| } |
| lassign [get_node_cpus [dict get $res_info $res_name "Nodes"]] cputot threadcnt |
| |
| # Delete the reservation |
| set ret_code [delete_res $res_name] |
| if {$ret_code != 0} { |
| fail "Unable to delete reservation ($res_name)" |
| } |
| |
| set nodes [get_nodes_by_state] |
| set num_nodes [llength $nodes] |
| |
| set core_res_num [ expr $cores_per_node / 2 ] |
| set thread_res_num [ expr $core_res_num * $threadcnt ] |
| set job_id 0 |
| |
| # Submit a batch job using half the threads on the nodes |
| spawn $sbatch -w[node_list_to_range $nodes] --time=10:00 --ntasks-per-node=$thread_res_num --output=/dev/null $file_in |
| expect { |
| -re "Submitted batch job ($number)" { |
| set job_id $expect_out(1,string) |
| exp_continue |
| } |
| timeout { |
| cancel_job $job_id |
| fail "sbatch not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| if {$job_id == 0} { |
| fail "Batch submit failure" |
| } |
| |
| subtest {[wait_for_job $job_id "RUNNING"] == 0} "A batch job using half the threads on the nodes should run" |
| |
| spawn $scontrol show job $job_id |
| expect { |
| timeout { |
| cancel_job $job_id |
| fail "scontrol not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| |
| # Make a reservation using 1 core per node in first 5 nodes |
| if {[expr $max_node_inx_int - $min_node_inx_int] != 4} { |
| subskip "Insufficient node count for remaining subtests (needs at least 5)" |
| cancel_job $job_id |
| return |
| } |
| |
| set ret_code [create_res $res_name "StartTime=now Duration=60 Nodes=$def_node_name\[$min_node_inx\-$max_node_inx\] CoreCnt=5 User=$user_name"] |
| if {$ret_code != 0} { |
| cancel_job $job_id |
| fail "Unable to create a valid reservation" |
| } |
| |
| # Get reservation info |
| set res_info [get_reservations $res_name] |
| if { ![dict exists $res_info $res_name] } { |
| delete_res $res_name |
| cancel_job $job_id |
| fail "Unable to get info about reservation ($res_name)" |
| } |
| lassign [get_node_cpus [dict get $res_info $res_name "Nodes"]] cputot threadcnt |
| |
| set res_nodecnt [dict get $res_info $res_name "NodeCnt"] |
| subtest {$res_nodecnt == 5} "Reservation should be created with 5 nodes" "Reservation was created with $res_nodecnt nodes" |
| |
| set res_corecnt [dict get $res_info $res_name "CoreCnt"] |
| subtest {$res_corecnt == 5} "Reservation should be created with 5 cores" "Reservation was created with $res_corecnt cores" |
| |
| # Delete the reservation |
| set ret_code [delete_res $res_name] |
| if {$ret_code != 0} { |
| cancel_job $job_id |
| fail "Unable to delete reservation ($res_name)" |
| } |
| |
| # Get the core_cnt left + 1 from the job above. |
| set core_cnt [expr (($cores_per_node / 2) * 5) + 1] |
| # Make the reservation using more cores then free in a node |
| set ret_code [create_res $res_name "StartTime=now Duration=60 Nodes=$def_node_name\[$min_node_inx\-$max_node_inx\] CoreCnt=$core_cnt User=$user_name"] |
| if {! [ |
| subtest {$ret_code != 0} "A reservation using more cores than free in a node should not be created" |
| ]} { |
| delete_res $res_name |
| } |
| |
| # Make the reservation using more cores than free in a node (now) |
| # but those cores being free at reservation start time |
| set ret_code [create_res $res_name "StartTime=now+3600 Duration=60 Nodes=$def_node_name\[$min_node_inx\-$max_node_inx\] CoreCnt=$core_cnt User=$user_name"] |
| subtest {$ret_code == 0} "A reservation using more cores than currently free but available at reservation start time should succeed" |
| |
| # Delete the reservation |
| set ret_code [delete_res $res_name] |
| if {$ret_code != 0} { |
| cancel_job $job_id |
| fail "Unable to delete reservation ($res_name)" |
| } |
| |
| # Make the reservation using more cores than free at reservation start time |
| set ret_code [create_res $res_name "StartTime=now+300 Duration=60 Nodes=$def_node_name\[$min_node_inx\-$max_node_inx\] CoreCnt=$core_cnt User=$user_name"] |
| if {! [ |
| subtest {$ret_code != 0} "A reservation using more cores than free at reservation start time should not be created" |
| ]} { |
| delete_res $res_name |
| } |
| |
| cancel_job $job_id |
| |
| log_info "Let's check overlapping reservations" |
| |
| set core_res_num [ expr $cores_per_node / 2 ] |
| set total_core_res $core_res_num |
| # Make a reservation for all nodes using just half the processor in each node |
| set ret_code [create_res $res_name "StartTime=now Duration=60 Nodes=$def_node_name\[$min_node_inx\] CoreCnt=$total_core_res User=$user_name"] |
| if {$ret_code != 0} { |
| fail "Unable to create a valid reservation" |
| } |
| |
| log_info "Reservation was created as expected" |
| |
| |
| if {$core_res_num < 2} { |
| log_warn "Not enough cores for remaining tests" |
| if [delete_res $res_name] { |
| fail "Unable to delete reservation ($res_name)" |
| } |
| return |
| } |
| |
| set core_cnt [expr $total_core_res + 1] |
| # Now creating a reservation using first node and more cores per node than available |
| set ret_code [create_res $res_name_test "StartTime=now Duration=60 Nodes=$def_node_name\[$min_node_inx\] CoreCnt=$core_cnt User=$user_name"] |
| if {! [ |
| subtest {$ret_code != 0} "A reservation using first node and more cores per node than available should not be created" |
| ]} { |
| delete_res $res_name_test |
| } |
| |
| # Now creating a reservation using first node and just 1 core per node |
| set ret_code [create_res $res_name_test "StartTime=now Duration=60 Nodes=$def_node_name\[$min_node_inx\] CoreCnt=1 User=$user_name"] |
| if {$ret_code != 0} { |
| delete_res $res_name |
| fail "Unable to create a reservation using first 5 nodes and just 1 core per node" |
| } |
| |
| log_info "Reservation was created as expected" |
| |
| # Get reservation info (first reservation) |
| set res_info [get_reservations $res_name] |
| if { ![dict exists $res_info $res_name] } { |
| delete_res $res_name |
| delete_res $res_name_test |
| fail "Unable to get info about reservation ($res_name)" |
| } |
| lassign [get_node_cpus [dict get $res_info $res_name "Nodes"]] cputot threadcnt |
| |
| # Submit a batch job: a job using cores available in first node |
| set core_res_num [ expr $cores_per_node / 2 ] |
| set core_res_num [ expr $core_res_num - 1 ] |
| set thread_res_num [ expr $core_res_num * $threadcnt ] |
| |
| spawn $sbatch --ntasks-per-node=$thread_res_num --nodelist=$def_node_name\[$min_node_inx\] --output=/dev/null $file_in |
| expect { |
| -re "Submitted batch job ($number)" { |
| set job_id $expect_out(1,string) |
| exp_continue |
| } |
| timeout { |
| delete_res $res_name |
| delete_res $res_name_test |
| fail "sbatch not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| if {$job_id == 0} { |
| delete_res $res_name |
| delete_res $res_name_test |
| fail "Batch submit failure" |
| } |
| |
| sleep 10 |
| |
| # Show the job, make sure reservation tag is right |
| spawn $scontrol show job $job_id |
| expect { |
| -re "Invalid job id specified" { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "Job ($job_id) not found" |
| } |
| -re "JobState=PENDING" { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "Job ($job_id) is PENDING" |
| } |
| timeout { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "scontrol not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| |
| cancel_job $job_id |
| |
| log_debug "JOB is running as expected" |
| |
| # Submit a batch job: a job using more cores than available in first node |
| set core_res_num [ expr $cores_per_node / 2 ] |
| set thread_res_num [ expr $core_res_num * $threadcnt ] |
| spawn $sbatch --ntasks-per-node=$thread_res_num --nodelist=$def_node_name\[$min_node_inx\] --output=/dev/null $file_in |
| expect { |
| -re "Submitted batch job ($number)" { |
| set job_id $expect_out(1,string) |
| exp_continue |
| } |
| timeout { |
| delete_res $res_name |
| delete_res $res_name_test |
| fail "sbatch not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| if {$job_id == 0} { |
| delete_res $res_name |
| delete_res $res_name_test |
| fail "Batch submit failure" |
| } |
| |
| sleep 10 |
| |
| # Show the job, make sure reservation tag is right |
| spawn $scontrol show job $job_id |
| expect { |
| -re "Invalid job id specified" { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "Job ($job_id) not found" |
| } |
| -re "JobState=PENDING" { |
| log_debug "Job is PENDING as expected" |
| exp_continue |
| } |
| -re "JobState=RUNNING" { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "Job ($job_id) is RUNNING" |
| } |
| timeout { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "scontrol not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| |
| cancel_job $job_id |
| |
| # Submit a batch job: a job using cores reserved in first reservation |
| set core_res_num [ expr $cores_per_node / 2 ] |
| set thread_res_num [ expr $core_res_num * $threadcnt ] |
| spawn $sbatch --ntasks-per-node=$thread_res_num --nodelist=$def_node_name\[$min_node_inx\] --reservation=$res_name --output=/dev/null $file_in |
| expect { |
| -re "Submitted batch job ($number)" { |
| set job_id $expect_out(1,string) |
| exp_continue |
| } |
| timeout { |
| delete_res $res_name |
| delete_res $res_name_test |
| fail "sbatch not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| if {$job_id == 0} { |
| delete_res $res_name |
| delete_res $res_name_test |
| fail "Batch submit failure" |
| } |
| |
| sleep 10 |
| |
| # Show the job, make sure reservation tag is right |
| spawn $scontrol show job $job_id |
| expect { |
| -re "Invalid job id specified" { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "Job ($job_id) not found" |
| } |
| -re "JobState=RUNNING" { |
| log_debug "Job is RUNNING as expected" |
| exp_continue |
| } |
| -re "JobState=PENDING" { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "Job ($job_id) is PENDING" |
| } |
| timeout { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "scontrol not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| |
| cancel_job $job_id |
| |
| # Submit a batch job: a job using more cores than reserved in first reservation |
| set core_res_num [ expr $cores_per_node / 2 ] |
| set core_res_num [ expr $core_res_num + 1 ] |
| set thread_res_num [ expr $core_res_num * $threadcnt ] |
| spawn $sbatch --ntasks-per-node=$thread_res_num --nodelist=$def_node_name\[$min_node_inx\] --reservation=$res_name --output=/dev/null $file_in |
| expect { |
| -re "Submitted batch job ($number)" { |
| set job_id $expect_out(1,string) |
| exp_continue |
| } |
| timeout { |
| delete_res $res_name |
| delete_res $res_name_test |
| fail "sbatch not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| if {$job_id == 0} { |
| delete_res $res_name |
| delete_res $res_name_test |
| fail "Batch submit failure" |
| } |
| |
| sleep 10 |
| |
| # Show the job, make sure reservation tag is right |
| spawn $scontrol show job $job_id |
| expect { |
| -re "Invalid job id specified" { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "Job ($job_id) not found" |
| } |
| -re "JobState=PENDING" { |
| log_debug "Job is PENDING as expected" |
| exp_continue |
| } |
| -re "JobState=RUNNING" { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "Job ($job_id) is RUNNING" |
| } |
| timeout { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "scontrol not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| |
| cancel_job $job_id |
| |
| # Submit a batch job: a job using cores reserved in second reservation |
| set thread_res_num [ expr 1 * $threadcnt ] |
| spawn $sbatch --ntasks-per-node=$thread_res_num --nodelist=$def_node_name\[$min_node_inx\] --reservation=$res_name_test --output=/dev/null $file_in |
| expect { |
| -re "Submitted batch job ($number)" { |
| set job_id $expect_out(1,string) |
| exp_continue |
| } |
| timeout { |
| delete_res $res_name |
| delete_res $res_name_test |
| fail "sbatch not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| if {$job_id == 0} { |
| delete_res $res_name |
| delete_res $res_name_test |
| fail "Batch submit failure" |
| } |
| |
| sleep 10 |
| |
| # Show the job, make sure reservation tag is right |
| spawn $scontrol show job $job_id |
| expect { |
| -re "Invalid job id specified" { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "Job ($job_id) not found" |
| } |
| -re "JobState=RUNNING" { |
| log_debug "Job is RUNNING as expected" |
| exp_continue |
| } |
| -re "JobState=PENDING" { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "Job ($job_id) is PENDING" |
| } |
| timeout { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "scontrol not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| |
| cancel_job $job_id |
| |
| # Submit a batch job: a job using more cores than reserved in second reservation |
| set thread_res_num [ expr 2 * $threadcnt ] |
| spawn $sbatch --ntasks-per-node=$thread_res_num --nodelist=$def_node_name\[$min_node_inx\] --reservation=$res_name_test --output=/dev/null $file_in |
| expect { |
| -re "Submitted batch job ($number)" { |
| set job_id $expect_out(1,string) |
| exp_continue |
| } |
| timeout { |
| delete_res $res_name |
| delete_res $res_name_test |
| fail "sbatch not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| if {$job_id == 0} { |
| delete_res $res_name |
| delete_res $res_name_test |
| fail "Batch submit failure" |
| } |
| |
| sleep 10 |
| |
| # Show the job, make sure reservation tag is right |
| spawn $scontrol show job $job_id |
| expect { |
| -re "Invalid job id specified" { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "Job ($job_id) not found" |
| } |
| -re "JobState=PENDING" { |
| log_debug "Job is PENDING as expected" |
| exp_continue |
| } |
| -re "JobState=RUNNING" { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "Job ($job_id) is RUNNING" |
| } |
| timeout { |
| delete_res $res_name |
| delete_res $res_name_test |
| cancel_job $job_id |
| fail "scontrol not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| |
| cancel_job $job_id |
| |
| set ret_code1 [delete_res $res_name] |
| if { $ret_code1 != 0 } { |
| delete_res $res_name_test |
| fail "Failed to delete reservation ($res_name)" |
| } |
| set ret_code [delete_res $res_name_test] |
| if { $ret_code != 0 } { |
| fail "Failed to delete reservation ($res_name_test)" |
| } |
| |
| # Create a job that uses all cores in the node range and is not |
| # part of a reservation. |
| spawn $sbatch --ntasks-per-node=$cores_per_node --nodelist=$def_node_name\[$min_node_inx\] --output=/dev/null $file_in |
| expect { |
| -re "Submitted batch job ($number)" { |
| set job_id $expect_out(1,string) |
| exp_continue |
| } |
| timeout { |
| fail "sbatch not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| if {$job_id == 0} { |
| fail "Batch submit failure" |
| } |
| |
| sleep 10 |
| |
| # Verify the job is running. |
| set job_is_running false |
| spawn $scontrol show job $job_id |
| expect { |
| -re "Invalid job id specified" { |
| cancel_job $job_id |
| fail "Job ($job_id) not found" |
| } |
| -re "JobState=RUNNING" { |
| set job_is_running true |
| log_debug "Job is RUNNING as expected" |
| exp_continue |
| } |
| timeout { |
| cancel_job $job_id |
| fail "scontrol not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| if { ! $job_is_running } { |
| cancel_job $job_id |
| fail "Unable to verify the job is running" |
| } |
| |
| # Make the reservation using more cores than free, but use the |
| # IGNORE_JOBS flags. Verify that it is created with the |
| # correct nodes, CoreCnt, and CoreIDs. |
| set core_cnt [expr 8 + $cores_per_node] |
| set ret_code [create_res $res_name "StartTime=now Duration=60 Nodes=$def_node_name\[$min_node_inx\-$max_node_inx\] CoreCnt=$core_cnt User=$user_name Flags=IGNORE_JOBS"] |
| if {$ret_code == 1} { |
| cancel_job $job_id |
| fail "Reservation was not created when it should have been" |
| } |
| log_info "Reservation was created as expected" |
| |
| set exp_node_cnt [expr {$max_node_inx_int-$min_node_inx_int+1}] |
| set exp_core_cnt [expr {8+$cores_per_node}] |
| set hosts_correct 0 |
| set node_cnt_correct 0 |
| set core_cnt_correct 0 |
| set core_ids_correct 0 |
| |
| set node_inxs [node_range_to_list "$def_node_name\[$min_node_inx-$max_node_inx_int\]"] |
| |
| sleep 5 |
| |
| spawn $scontrol show res |
| expect { |
| -re "Nodes=$def_node_name\\\[$min_node_inx\-$max_node_inx" { |
| set hosts_correct 1 |
| exp_continue |
| } |
| -re "NodeCnt=$exp_node_cnt" { |
| set node_cnt_correct 1 |
| exp_continue |
| } |
| -re "CoreCnt=$exp_core_cnt" { |
| set core_cnt_correct 1 |
| exp_continue |
| } |
| -re "NodeName=$re_word_str CoreIDs=\($re_word_str\)" { |
| set core_inxs [range_to_list $expect_out(1,string)] |
| set core_ids_correct [expr $core_ids_correct + [llength $core_inxs]] |
| exp_continue |
| } |
| timeout { |
| cancel_job $job_id |
| delete_res $res_name |
| fail "scontrol not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| |
| cancel_job $job_id |
| set ret_code [delete_res $res_name] |
| if { $ret_code != 0 } { |
| fail "Error $ret_code deleting reservation" |
| } |
| |
| subtest {$hosts_correct == 1} "Reservation should have correct node list" |
| subtest {$node_cnt_correct == 1} "Reservation should have correct node count" |
| subtest {$core_cnt_correct == 1} "Reservation should have correct core count" |
| subtest {$core_ids_correct == $exp_core_cnt} "Reservation should have correct core ids $core_ids_correct ? $exp_core_cnt" |
| |
| # Create a job that runs on a single core on all nodes in the reservation. |
| |
| spawn $sbatch --ntasks-per-node=1 --nodelist=$def_node_name\[$min_node_inx\-$max_node_inx\] --output=/dev/null $file_in |
| expect { |
| -re "Submitted batch job ($number)" { |
| set job_id $expect_out(1,string) |
| exp_continue |
| } |
| timeout { |
| fail "sbatch not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| if {$job_id == 0} { |
| fail "Batch submit failure" |
| } |
| |
| sleep 10 |
| |
| spawn $scontrol show job $job_id |
| expect { |
| -re "Invalid job id specified" { |
| cancel_job $job_id |
| fail "Job ($job_id) not found" |
| } |
| -re "JobState=RUNNING" { |
| log_debug "Job is RUNNING as expected" |
| exp_continue |
| } |
| timeout { |
| cancel_job $job_id |
| fail "scontrol not responding" |
| } |
| eof { |
| wait |
| } |
| } |
| |
| cancel_job $job_id |
| } |
