| #!/usr/bin/env expect | |
| ############################################################################ | |
| # Purpose: Establish global state information for Slurm federation tests | |
| # | |
| # To define site-specific state information, set the values in a file | |
| # named 'globals.local'. Those values will override any specified here. | |
| # for example: | |
| # | |
| # $ cat globals.local | |
| # set slurm_dir "/usr/local" | |
| # set mpicc "/usr/local/bin/mpicc" | |
| # | |
| ############################################################################ | |
| # Copyright (C) SchedMD LLC. | |
| # This file is part of Slurm, a resource management program. | |
| # For details, see <https://slurm.schedmd.com/>. | |
| # Please also read the supplied file: DISCLAIMER. | |
| # | |
| # Slurm is free software; you can redistribute it and/or modify it under | |
| # the terms of the GNU General Public License as published by the Free | |
| # Software Foundation; either version 2 of the License, or (at your option) | |
| # any later version. | |
| # | |
| # Slurm is distributed in the hope that it will be useful, but WITHOUT ANY | |
| # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | |
| # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more | |
| # details. | |
| # | |
| # You should have received a copy of the GNU General Public License along | |
| # with SLURM; if not, write to the Free Software Foundation, Inc., | |
| # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | |
| ############################################################################ | |
| source ./globals | |
| # Set if testing federations | |
| cset fed_slurm_base "" | |
| cset fedc1 "" | |
| cset fedc2 "" | |
| cset fedc3 "" | |
| proc check_federation_setup { } { | |
| global fed_slurm_base fedc1 fedc2 fedc3 | |
| set rc true | |
| if {$fed_slurm_base eq "" || $fedc1 eq "" || $fedc2 eq "" || $fedc3 eq ""} { | |
| set rc false | |
| } | |
| return $rc | |
| } | |
| proc setup_federation { fed_name } { | |
| global sacctmgr fedc1 fedc2 fedc3 eol | |
| set rc $::RETURN_SUCCESS | |
| spawn $sacctmgr -i add federation $fed_name | |
| set matches 0 | |
| expect { | |
| -re "Adding Federation\\(s\\)$eol" { | |
| incr matches | |
| exp_continue | |
| } | |
| -re "$fed_name$eol" { | |
| incr matches | |
| exp_continue | |
| } | |
| timeout { | |
| fail "sacctmgr add not responding" | |
| } | |
| eof { | |
| wait | |
| } | |
| } | |
| if {!$rc && $matches != 2} { | |
| log_error "Failed to create federation" | |
| return $::RETURN_ERROR | |
| } | |
| set count 0 | |
| foreach cluster [list $fedc1 $fedc2 $fedc3] { | |
| incr count | |
| spawn $sacctmgr -i mod cluster $cluster set federation=$fed_name features= | |
| set matches 0 | |
| expect { | |
| -re "Setting$eol" { | |
| incr matches | |
| exp_continue | |
| } | |
| -re "^\\s+Feature\\s+=\\s+$eol" { | |
| incr matches | |
| exp_continue | |
| } | |
| -re "^\\s+Federation\\s+=\\s+$fed_name$eol" { | |
| incr matches | |
| exp_continue | |
| } | |
| -re "Modified cluster...$eol" { | |
| incr matches | |
| exp_continue | |
| } | |
| -re "^\\s+$cluster$eol" { | |
| incr matches | |
| exp_continue | |
| } | |
| timeout { | |
| fail "sacctmgr add not responding" | |
| } | |
| eof { | |
| wait | |
| } | |
| } | |
| if {!$rc && $matches != 5} { | |
| log_error "Failed to add $cluster to federation" | |
| set rc $::RETURN_ERROR | |
| break | |
| } | |
| if {$count > 1} { | |
| sleep 5 | |
| } | |
| } | |
| return $rc | |
| } | |
| proc check_cluster_up { cname } { | |
| set rc true | |
| set matches 0 | |
| set timeout 2 | |
| global fed_slurm_base fedc1 fedc2 fedc3 | |
| set my_scontrol "${fed_slurm_base}/$cname/bin/scontrol" | |
| log_user 0 | |
| set my_pid [spawn $my_scontrol show config] | |
| expect { | |
| "Configuration data as of" { | |
| incr matches | |
| } | |
| timeout { | |
| log_warn "$cname not responding" | |
| slow_kill $my_pid | |
| set rc false | |
| } | |
| eof { | |
| wait | |
| } | |
| } | |
| if {$matches != 1} { | |
| log_error "$cname not responding" | |
| set rc false | |
| } | |
| log_user 1 | |
| return $rc | |
| } | |
| proc check_federation_up {} { | |
| set rc true | |
| global fedc1 fedc2 fedc3 | |
| if {![check_cluster_up $fedc1] || | |
| ![check_cluster_up $fedc2] || | |
| ![check_cluster_up $fedc3]} { | |
| log_warn "This test can't be run if any clusters--$fedc1,\ | |
| $fedc2, or $fedc3--are down" | |
| set rc false | |
| } | |
| return $rc | |
| } | |
| proc delete_federations { names } { | |
| global sacctmgr | |
| set matches 0 | |
| set rc $::RETURN_SUCCESS | |
| set object "federation" | |
| spawn $sacctmgr -i delete $object $names | |
| expect { | |
| -re "privilege to perform this action" { | |
| log_error "Don't have privileges" | |
| set rc $::RETURN_ERROR | |
| } | |
| -re "(There was a problem|Unknown condition|Bad format on|Bad MaxWall|Unknown option)" { | |
| log_error "There was a problem with the sacctmgr command" | |
| set rc $::RETURN_ERROR | |
| } | |
| -re "Problem getting" { | |
| log_error "There was a problem getting information from the database" | |
| set rc $::RETURN_ERROR | |
| } | |
| -re "Problem adding" { | |
| log_error "There was an unknown problem" | |
| set rc $::RETURN_ERROR | |
| } | |
| -re "No associations" { | |
| log_error "Your command didn't return anything" | |
| set rc $::RETURN_ERROR | |
| } | |
| -re "Deleting $object" { | |
| incr matches | |
| exp_continue | |
| } | |
| -re " Nothing deleted" { | |
| incr matches | |
| exp_continue | |
| } | |
| timeout { | |
| fail "sacctmgr delete not responding" | |
| } | |
| eof { | |
| wait | |
| } | |
| } | |
| if {!$rc && $matches != 1} { | |
| log_error "sacctmgr had a problem deleting $object. Got $matches" | |
| } | |
| return $rc | |
| } | |
| proc get_clusterfed_info { fed_name } { | |
| global sacctmgr eol | |
| set matches 0 | |
| array set clusters {} | |
| spawn $sacctmgr show cluster federation=$fed_name \ | |
| format="cluster%20,federation%20,id,controlhost,controlport,features,fedstate" | |
| expect { | |
| -re "Cluster\\s+Federation\\s+ID\\s+ControlHost\\s+ControlPort\\s+Features\\s+FedState $eol" { | |
| incr matches | |
| exp_continue | |
| } | |
| -re "\\s+(\\S+)\\s+$fed_name\\s+(\\d+)\\s+(\\S+)\\s+(\\d+)\\s+(\\S*)\\s+(\\S*) $eol" { | |
| set clusters($expect_out(1,string)) [dict create id $expect_out(2,string) \ | |
| host $expect_out(3,string) \ | |
| port $expect_out(4,string) \ | |
| features $expect_out(5,string) \ | |
| state $expect_out(6,string)] | |
| incr matches | |
| exp_continue | |
| } | |
| timeout { | |
| fail "sacctmgr add not responding" | |
| } | |
| eof { | |
| wait | |
| } | |
| } | |
| if {$matches < 2} { | |
| fail "Didn't match enough clusters for federation ($fed_name) ($matches < 2)" | |
| } | |
| return [array get clusters] | |
| } | |
| # | |
| # Add a single cluster to the given federation. | |
| # IN: cname - name of cluster to add to federation. | |
| # IN: fed_name - name of federation to add cluster to. | |
| # RET: RETURN_SUCCESS, or non-zero on error. | |
| # | |
| proc add_cluster_to_fed {cname fed_name} { | |
| global sacctmgr eol | |
| set rc $::RETURN_SUCCESS | |
| set matches 0 | |
| spawn $sacctmgr -i modify federation $fed_name set clusters+=$cname | |
| expect { | |
| -re "Setting$eol" { | |
| incr matches | |
| exp_continue | |
| } | |
| -re "Cluster\\s+ \\+= $cname$eol" { | |
| incr matches | |
| exp_continue | |
| } | |
| -re "^\\s+Modified federation...$eol" { | |
| incr matches | |
| exp_continue | |
| } | |
| -re "\\s+$fed_name$eol" { | |
| incr matches | |
| exp_continue | |
| } | |
| timeout { | |
| fail "sacctmgr add not responding" | |
| } | |
| eof { | |
| wait | |
| } | |
| } | |
| if {$rc || $matches != 4} { | |
| log_error "Failed to add $cname to $fed_name ($matches != 4)" | |
| set $rc $::RETURN_ERROR | |
| } | |
| return $rc | |
| } | |
| # | |
| # Remove a single cluster from the given federation. | |
| # IN: cname - name of cluster to remove from the federation. | |
| # IN: fed_name - name of federation to remove cluster from. | |
| # RET: RETURN_SUCCESS, or non-zero on error. | |
| # | |
| proc remove_cluster_from_fed {cname fed_name} { | |
| global sacctmgr eol | |
| set rc $::RETURN_SUCCESS | |
| set matches 0 | |
| spawn $sacctmgr -i modify federation $fed_name set clusters-=$cname | |
| expect { | |
| -re "Setting$eol" { | |
| incr matches | |
| exp_continue | |
| } | |
| -re "Cluster\\s+ -= $cname$eol" { | |
| incr matches | |
| exp_continue | |
| } | |
| -re "^\\s+Modified federation...$eol" { | |
| incr matches | |
| exp_continue | |
| } | |
| -re "\\s+$fed_name$eol" { | |
| incr matches | |
| exp_continue | |
| } | |
| timeout { | |
| fail "sacctmgr add not responding" | |
| } | |
| eof { | |
| wait | |
| } | |
| } | |
| if {$rc || $matches != 4} { | |
| log_error "Failed to remove $cname from $fed_name" | |
| set $rc $::RETURN_ERROR | |
| } | |
| return $rc | |
| } | |
| ################################################################ | |
| # | |
| # NAME | |
| # wait_for_fed_job - waits for a job to reach the desired state | |
| # | |
| # SYNOPSIS | |
| # wait_for_fed_job ?options? job_id desired_state clusters | |
| # | |
| # DESCRIPTION | |
| # Wait for a previously submitted Slurm job to reach the desired state. | |
| # | |
| # OPTIONS | |
| # -timeout <integer_number> | |
| # time in seconds to wait for the job to be in the desired state | |
| # before timing out (default is 360) | |
| # -pollinterval <integer_number> | |
| # time in seconds between each job state check (default is 1) | |
| # | |
| # ARGUMENTS | |
| # job_id | |
| # The Slurm job id of a job we want to wait for. | |
| # desired_state | |
| # The state you want the job to attain before | |
| # returning. Currently supports: | |
| # DONE any terminated state | |
| # PENDING job is pending | |
| # RUNNING job is running | |
| # SUSPENDED job is suspended | |
| # clusters | |
| # The list of clusters to wait for. If empty "" the default | |
| # clusters of fedc1, fedc2 and fedc3 will be used. | |
| # | |
| # RETURN VALUE | |
| # The name of the first cluster on which the job is found | |
| # in the desired state. Empty string indicates a failure. | |
| # | |
| # NOTE | |
| # We sleep for two seconds before replying that a job is | |
| # done to give time for I/O completion (stdout/stderr files) | |
| # | |
| ################################################################ | |
| proc wait_for_fed_job args { | |
| global scontrol fedc1 fedc2 fedc3 | |
| set timeout 360 | |
| set poll_interval 1 | |
| while {[llength $args]} { | |
| switch -glob -- [lindex $args 0] { | |
| -time* {set args [lassign $args - timeout]} | |
| -poll* {set args [lassign $args - poll_interval]} | |
| -* {fail "Unknown option: [lindex $args 0]"} | |
| default break | |
| } | |
| } | |
| set argument_count [llength $args] | |
| if {$argument_count != 3} { | |
| fail "Invalid number of arguments ($argument_count): $args" | |
| } else { | |
| lassign $args job_id desired_state clusters | |
| } | |
| # First verify that desired_state is supported | |
| switch $desired_state { | |
| "DONE" {} | |
| "PENDING" {} | |
| "REVOKED" {} | |
| "RUNNING" {} | |
| "SUSPENDED" {} | |
| "SPECIAL_EXIT" {} | |
| default { | |
| log_error "Invalid desired state: $desired_state" | |
| return "" | |
| } | |
| } | |
| if {$job_id == 0} { | |
| log_error "Invalid job ID: $job_id" | |
| return "" | |
| } | |
| set my_delay 0 | |
| set spec_clusters [list $fedc1 $fedc2 $fedc3] | |
| if {$clusters ne ""} { | |
| set spec_clusters [split $clusters ","] | |
| } | |
| log_debug "Checking for job '$job_id' in state '$desired_state' on [join $spec_clusters ,]" | |
| while 1 { | |
| foreach cluster $spec_clusters { | |
| log_debug "Checking $cluster" | |
| set fd [open "|$scontrol -M$cluster --local -a -o show job $job_id"] | |
| gets $fd line | |
| catch {close $fd} | |
| if {[regexp {JobState\s*=\s*(\w+)} $line foo state] != 1} { | |
| log_warn "Unable to determine the state of job $job_id on cluster $cluster" | |
| continue | |
| } | |
| switch $state { | |
| "NOT_FOUND" - | |
| "CANCELLED" - | |
| "DEADLINE" - | |
| "FAILED" - | |
| "TIMEOUT" - | |
| "NODE_FAIL" - | |
| "PREEMPTED" - | |
| "COMPLETED" { | |
| if {$desired_state eq "DONE"} { | |
| log_debug "Job $job_id is DONE ($state) on $cluster" | |
| sleep 2 | |
| return $cluster | |
| } | |
| if {$desired_state eq "RUNNING"} { | |
| log_warn "Job $job_id is $state but we wanted RUNNING" | |
| } | |
| if {$desired_state eq "SUSPENDED"} { | |
| log_warn "Job $job_id is $state but we wanted SUSPENDED" | |
| } | |
| return "" | |
| } | |
| "PENDING" { | |
| if {$desired_state eq "PENDING"} { | |
| log_debug "Job $job_id is PENDING on $cluster" | |
| return $cluster | |
| } | |
| log_warn "Job $job_id is in state $state, but desired state $desired_state" | |
| } | |
| "REVOKED" { | |
| if {$desired_state eq "REVOKED"} { | |
| log_debug "Job $job_id is REVOKED on $cluster" | |
| return $cluster | |
| } | |
| log_warn "Job $job_id is in state $state, but desired state $desired_state" | |
| } | |
| "RUNNING" { | |
| if {$desired_state eq "RUNNING"} { | |
| log_debug "Job $job_id is RUNNING on $cluster" | |
| return $cluster | |
| } | |
| log_warn "Job $job_id is in state $state, but desired state $desired_state" | |
| } | |
| "SPECIAL_EXIT" { | |
| if {$desired_state eq "SPECIAL_EXIT"} { | |
| log_debug "Job $job_id is SPECIAL_EXIT on $cluster" | |
| return $cluster | |
| } | |
| log_warn "Job $job_id is in state $state, but desired state $desired_state" | |
| } | |
| "SUSPENDED" { | |
| if {$desired_state eq "SUSPENDED"} { | |
| log_debug "Job $job_id is SUSPENDED on $cluster" | |
| return $cluster | |
| } | |
| log_warn "Job $job_id is in state $state, but desired state $desired_state" | |
| } | |
| default { | |
| log_debug "Job $job_id is in state $state. Desired state was $desired_state" | |
| } | |
| } | |
| } | |
| if { $my_delay > $timeout } { | |
| log_error "Timeout waiting for job state $desired_state" | |
| return "" | |
| } | |
| exec sleep $poll_interval | |
| set my_delay [expr $my_delay + $poll_interval] | |
| } | |
| } |