blob: 66ea76c67afe775bde88d5fb48c3940006342468 [file] [log] [blame] [edit]
#!/usr/bin/expect
############################################################################
# Purpose: Test of SLURM functionality
# Test of sinfo cpu total and allocated
#
# Output: "TEST: #.#" followed by "SUCCESS" if test was successful, OR
# "FAILURE: ..." otherwise with an explanation of the failure, OR
# anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2009 Lawrence Livermore National Security.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Joseph Donaghy <donaghy1@llnl.gov>
# CODE-OCEC-09-009. All rights reserved.
#
# This file is part of SLURM, a resource management program.
# For details, see <http://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
############################################################################
source ./globals
set test_id "4.12"
set file_in "test$test_id.in"
set exit_code 0
set pnumsc 0
set pnumsi 0
set aprocsc 0
set aprocsi 0
set inode_found 0
set inode_name ""
set smallest 1
set layout "static"
set select_type ""
set node_scaling 0
proc scontrol_test { node proc_cnt } {
global scontrol number prompt
upvar spawn_id spawn_id
set found 0
set rc 0
send "$scontrol show node $node\r"
expect {
-re "CPUAlloc=($number)" {
set num_alloc $expect_out(1,string)
set found 1
if {$proc_cnt != $num_alloc} {
send_user "\nFAILURE: requested $proc_cnt but got $num_alloc instead\n"
set rc 1
}
exp_continue
}
-re $prompt {
}
timeout {
send_user "\nFAILURE: scontrol not responding\n"
slow_kill $mypid
return 1
}
eof {
wait
}
}
if {!$found} {
send_user "\nFAILURE: didn't get expected output from scontrol\n"
set rc 1
}
return $rc
}
proc sinfo_test_1 { node proc_cnt total_procs idle_cpus } {
global float number prompt sinfo slash
upvar spawn_id spawn_id
set found 0
set rc 0
send "$sinfo -o \"%C %A %N\" -h -n $node\r"
# make sure we get by the sinfo command so we don't
# think the %'s are a prompt
expect {
"%C %A %N" {
}
}
expect {
-re "($float)(K?)($slash)($float)(K?)($slash)($float)(K?)($slash)($float)(K?) ($number)($slash)($number) $node" {
set found 1
set num_alloc $expect_out(1,string)
if {[string compare $expect_out(2,string) ""]} {
set num_alloc [expr $num_alloc * 1024]
}
set num_idle $expect_out(4,string)
if {[string compare $expect_out(5,string) ""]} {
set num_idle [expr $num_idle * 1024]
}
set num_other $expect_out(7,string)
if {[string compare $expect_out(8,string) ""]} {
set num_other [expr $num_other * 1024]
}
set num_total $expect_out(10,string)
if {[string compare $expect_out(11,string) ""]} {
set num_total [expr $num_total * 1024]
}
if { $num_alloc != $proc_cnt } {
send_user "\nFAILURE: sinfo 1 allocated cpus wrong, got $num_alloc but needed $proc_cnt\n"
set rc 1
} elseif { $num_idle != $idle_cpus } {
send_user "\nFAILURE: sinfo 1 idle cpus wrong, got $num_idle but needed $idle_cpus\n"
set rc 1
} elseif { $num_total != $total_procs } {
send_user "\nFAILURE: sinfo 1 total cpus wrong, got $num_total but needed $total_procs\n"
set rc 1
}
exp_continue
}
-re $prompt {
}
timeout {
send_user "\nFAILURE: sinfo not responding\n"
slow_kill $mypid
return 1
}
eof {
wait
}
}
if {!$found} {
send_user "\nFAILURE: didn't get expected output from sinfo\n"
set rc 1
}
return $rc
}
proc sinfo_test_2 { node proc_cnt total_procs } {
global sinfo number prompt node_scaling
upvar spawn_id spawn_id
set rc 0
set num_alloc 0
set num_idle 0
set alloc_nodes 1
set total_nodes 1
if {$node_scaling} {
set alloc_nodes [expr $proc_cnt / $node_scaling]
set total_nodes [expr $total_procs / $node_scaling]
}
set idle_nodes [expr $total_nodes - $alloc_nodes]
send "$sinfo -o \"%t %D %N\" -h -n $node\r"
# make sure we get by the sinfo command so we don't
# think the %'s are a prompt
expect {
"%t %D %N" {
}
}
expect {
-re "alloc ($number)(K?) $node" {
set num_alloc $expect_out(1,string)
if {[string compare $expect_out(2,string) ""]} {
set num_alloc [expr $inode_procs * 1024]
}
exp_continue
}
-re "idle ($number)(K?) $node" {
set num_idle $expect_out(1,string)
if {[string compare $expect_out(2,string) ""]} {
set num_idle [expr $num_idle * 1024]
}
exp_continue
}
-re "mix ($number)(K?) $node" {
set num_alloc $expect_out(1,string)
if {[string compare $expect_out(2,string) ""]} {
set num_alloc [expr $inode_procs * 1024]
}
exp_continue
}
-re $prompt {
}
timeout {
send_user "\nFAILURE: sinfo not responding\n"
slow_kill $mypid
return 1
}
eof {
wait
}
}
if { $num_alloc != $alloc_nodes } {
send_user "\nFAILURE: sinfo 2 allocated nodes wrong, got $num_alloc but needed $alloc_nodes\n"
set rc 1
} elseif { $num_idle != $idle_nodes } {
send_user "\nFAILURE: sinfo 2 idle nodes wrong, got $num_idle but needed $idle_nodes\n"
set rc 1
}
return $rc
}
# allocate a set of nodes (node_cnt) and the quit right after
proc allocate_and_quit { node proc_cnt total_procs } {
global bin_bash salloc scontrol sinfo number alpha_numeric_under
global prompt select_type procs_per_node test_id bin_rm
set file_in "test$test_id.input"
set job_id 0
set num_alloc 0
set block ""
set rc 0
set timeout 60
set idle_cpus [expr $total_procs - $proc_cnt]
make_bash_script $file_in "
export PS1=\"$prompt\"
$bin_bash --norc
"
set mypid [spawn $salloc -w $node -N 1 -t 5 -n $proc_cnt ./$file_in]
expect {
-re "Granted job allocation ($number)" {
set job_id $expect_out(1,string)
exp_continue
}
-re $prompt {
# test for scontrol to give me the correct cpu count
if { [scontrol_test $node $proc_cnt] } {
send "exit\r"
return 1
}
# test for sinfo to give me the correct cpu count
if { [sinfo_test_1 $node $proc_cnt $total_procs $idle_cpus] } {
send "exit\r"
return 1
}
# test for sinfo to give me the correct node count
if { [sinfo_test_2 $node $proc_cnt $total_procs] } {
send "exit\r"
return 1
}
send "exit\r"
exp_continue
}
-re "Unable to contact" {
send_user "\nFAILURE: slurm appears to be down\n"
exp_continue
}
timeout {
send_user "\nFAILURE: salloc not responding\n"
if {$job_id != 0} {
cancel_job $job_id
}
slow_kill $mypid
return 1
}
eof {
wait
}
}
exec $bin_rm -f $file_in
return $rc
}
############################################################################
# test starts here
############################################################################
print_header $test_id
# find the default partition
set def_part [default_partition]
# find the nodes in the default partition
log_user 0
set def_hostlist ""
set part_exclusive 0
spawn $scontrol show part $def_part
expect {
-re " Nodes=($alpha_numeric_nodelist)" {
set def_hostlist $expect_out(1,string)
exp_continue
}
-re " BasePartitions=($alpha_numeric_nodelist)" {
set def_hostlist $expect_out(1,string)
exp_continue
}
-re " Shared=EXCLUSIVE" {
set part_exclusive 1
exp_continue
}
timeout {
send_user "\nFAILURE: scontrol not responding\n"
set exit_code 1
}
eof {
wait
}
}
set host_cnt 0
spawn $scontrol show hostnames $def_hostlist
expect {
-re "($alpha_numeric_under)" {
set host_name($host_cnt) $expect_out(1,string)
incr host_cnt
exp_continue
}
timeout {
send_user "\nFAILURE: scontrol not responding\n"
set exit_code 1
}
eof {
wait
}
}
log_user 1
if {$host_cnt == 0} {
send_user "\nFAILURE: could not find any nodes in default partition\n"
exit 1
}
# find me an idle node in default partition
log_user 0
set inode_name ""
set inode_cores_per_socket 0
set inode_procs 0
set units ""
set inode_sockets 0
set inode_threads_per_core 0
set fd [open "|$scontrol --oneliner show node $def_hostlist"]
exp_internal 1
while {[gets $fd line] != -1} {
if {[regexp {NodeName=(\w+).*CoresPerSocket=(\d+).*CPUTot=(\d+)(K?).*Sockets=(\d+).*State=IDLE ThreadsPerCore=(\d+)} $line frag inode_name inode_cores_per_socket inode_procs units inode_sockets inode_threads_per_core] == 1} {
break
}
if {[regexp {NodeName=(\w+).*CoresPerSocket=(\d+).*CPUTot=(\d+)(K?).*Sockets=(\d+).*State=IDLE.CLOUD ThreadsPerCore=(\d+)} $line frag inode_name inode_cores_per_socket inode_procs units inode_sockets inode_threads_per_core] == 1} {
break
}
}
exp_internal 0
if {[string compare $units ""]} {
set inode_procs [expr $inode_procs * 1024]
}
catch {close $fd}
log_user 1
if {!$inode_procs} {
send_user "\nFAILURE: couldn't find an idle node in the default partition\n"
exit 1
}
send_user "found idle node $inode_name with $inode_procs processors\n"
# figure out the select plugin we are using
set select_type [test_select_type]
if {![string compare $select_type "bluegene"]} {
# figure out some things if a bluegene system
set layout [get_bluegene_layout]
if {$layout == 0} {
send_user "\nFAILURE: No layout mode found for this system\n"
exit 1
}
set psets [get_bluegene_psets]
if {$psets == 0} {
send_user "\nFAILURE: No psets are set on this system\n"
exit 1
}
set type [get_bluegene_type]
if {$type == 0} {
send_user "\nFAILURE: No bluegene type found \n"
exit 1
}
if {![string compare $type "Q"]} {
if {$psets >= 16} {
set smallest 32
} elseif {$psets >= 8} {
set smallest 64
} else {
set smallest 128
}
} elseif {![string compare $type "P"]} {
if {$psets >= 32} {
set smallest 16
} elseif {$psets >= 16} {
set smallest 32
} elseif {$psets >= 8} {
set smallest 64
} else {
set smallest 128
}
} elseif {![string compare $type "L"]} {
if {$psets >= 16} {
set smallest 32
} else {
set smallest 128
}
} else {
send_user "\nFAILURE: unknown bluegene system type '$type'\n";
exit 1
}
set node_scaling [get_bluegene_procs_per_cnode]
set smallest [expr $smallest * $node_scaling]
} elseif {![string compare $select_type "cray"]} {
set smallest $inode_procs
} elseif {![string compare $select_type "linear"]} {
set smallest $inode_procs
} else {
set select_params [test_select_type_params]
if {$part_exclusive == 1} {
set smallest $inode_procs
} elseif {![string compare $select_params "CR_CPU"]} {
set smallest $inode_threads_per_core
} elseif {![string compare $select_params "CR_CPU_MEMORY"]} {
set smallest $inode_threads_per_core
} elseif {![string compare $select_params "NONE"]} {
set smallest $inode_threads_per_core
} elseif {![string compare $select_params "CR_CORE"]} {
set smallest $inode_threads_per_core
} elseif {![string compare $select_params "CR_CORE_MEMORY"]} {
set smallest $inode_threads_per_core
} elseif {![string compare $select_params "CR_SOCKET"]} {
set smallest [expr $inode_cores_per_socket *$inode_threads_per_core]
} elseif {![string compare $select_params "CR_SOCKET_MEMORY"]} {
set smallest [expr $inode_cores_per_socket *$inode_threads_per_core]
} else {
send_user "\nWARNING: failed to parse SelectTypeParameters '$select_params'\n"
set smallest $inode_procs
}
}
set exit_code [allocate_and_quit $inode_name $smallest $inode_procs]
if {!$exit_code && $smallest != $inode_procs} {
# just to make sure we get a clean state we will sleep a bit
sleep 1
set exit_code [allocate_and_quit $inode_name $inode_procs $inode_procs]
}
if {$exit_code == 0} {
send_user "\nSUCCESS\n"
} else {
exit $exit_code
}