#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
#          Validate requeue'ing of federated jobs.
############################################################################
# Copyright (C) SchedMD LLC.
#
# This file is part of Slurm, a resource management program.
# For details, see <http://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals
source ./globals_accounting
source ./globals_federation

set long_script         "$test_dir/long"
set complete_script     "$test_dir/bash"
set exit_script         "$test_dir/exit"
set exithold_script     "$test_dir/exithold"
set fail_script         "$test_dir/fail"
set file_out            "$test_dir/output"
set prolog_script       "$test_dir/prolog"
set prologctl_script    "$test_dir/prologctl"
set epilog_script       "$test_dir/epilog"
set fed_name            "feda"
set long_running_job_id ""
set long_running_job_id2 ""
set job_id              0
set user_name           ""
set origin_cluster      ""
set non_origin_clusters ""
set dbd_delay           10

#
# Check accounting config and bail if not found.
#
if {[get_config_param "AccountingStorageType"] ne "accounting_storage/slurmdbd"} {
	skip "This test can't be run without a usable AccountStorageType"
}

if {[get_admin_level] ne "Administrator"} {
	skip "This test can't be run without being an Accounting administrator. Use: sacctmgr mod user \$USER set admin=admin"
}

regexp "($number)" [get_config_param "MinJobAge"] {} min_age
if {$min_age < 10} {
	skip "MinJobAge too low for this test ($min_age < 10)"
}

if {![check_federation_setup]} {
	skip "This test can't be run without fed_slurm_base, fedc1, fedc2, fedc3 setup in globals.local"
}

if {![check_federation_up]} {
	skip "This test can't be run without all clusters up"
}

proc cancel_federation_jobs { } {
	global bin_sleep scancel user_name fedc1 fedc2 fedc3

	spawn $scancel -M$fedc1,$fedc2,$fedc3 --user $user_name
	expect {
		eof {
			wait
		}
	}
	sleep 5
}

proc cleanup { } {
	global fed_name

	cancel_federation_jobs
	delete_federations $fed_name
}

proc check_ctl_state { job_id state cluster } {
	global scontrol

	set job_state 0
	spawn $scontrol -M$cluster -a --local show job $job_id
	expect {
		-re "JobState=$state" {
			set job_state 1
			exp_continue
		}
		timeout {
			fail "scontrol is not responding"
		}
		eof {
			wait
		}
	}

	if {$job_state != 1} {
		fail "Job ($job_id) state was not $state on cluster ($cluster)"
	}
}

proc check_missing_job { job_id cluster } {
	global scontrol

	set matched 0
	spawn $scontrol -M$cluster -a --local show job $job_id
	expect {
		"slurm_load_jobs error: Invalid job id specified" {
			set matched 1
			exp_continue
		}
		timeout {
			fail "scontrol is not responding"
		}
		eof {
			wait
		}
	}

	if {$matched != 1} {
		fail "Found an actual job ($job_id) on cluster ($cluster). This is not supposed to happen"
	}
}

# Count the number of jobs and steps with a specific job ID and state
# NOTE: Skip "extern" job container optionally spawned by "PrologFlags=contain"
proc check_dbd_states { job_id states cluster min_cnt } {
	global sacct

	set state_num 0
	spawn $sacct -M$cluster --job=$job_id --duplicates --parsable2 --start=now-15minutes --noheader -o JobID,State
	expect {
		-re "(\[0-9_\\.a-z\]+)\\|($states)" {
			if {[string first "extern" $expect_out(1,string)] == -1} {
				incr state_num
			}
			exp_continue
		}
		timeout {
			fail "sacct is not responding"
		}
		eof {
			wait
		}
	}

	if {$state_num < $min_cnt} {
		fail "Didn't find expected count $min_cnt (>$state_num) with state '$states' for job ($job_id) on cluster ($cluster)"
	}

	return 0
}

proc requeue_job { id } {

	global scontrol

	spawn $scontrol requeue $id
	expect {
		timeout {
			fail "scontrol is not responding"
		}
		eof {
			wait
		}
	}
}


proc requeuehold_job { id } {
	global scontrol

	spawn $scontrol requeuehold $id
	expect {
		timeout {
			fail "scontrol is not responding"
		}
		eof {
			wait
		}
	}
}

proc requeuehold_se_job { id } {
	global scontrol

	spawn $scontrol requeuehold state=specialexit $id
	expect {
		timeout {
			fail "scontrol is not responding"
		}
		eof {
			wait
		}
	}
}

proc release_job { id } {
	global scontrol

	spawn $scontrol release $id
	expect {
		timeout {
			fail "scontrol is not responding"
		}
		eof {
			wait
		}
	}
}

proc check_hold { job } {
	global scontrol number

	set matches 0
	spawn $scontrol -a --local show job $job
	expect {
		"Priority=0" {
			incr matches
			exp_continue
		}
		"Reason=job_requeued_in_held_state" {
			incr matches
			exp_continue
		}
		timeout {
			fail "scontrol is not responding"
		}
		eof {
			wait
		}
	}

	if { $matches != 2 } {
		fail "Priority was not set to a non zero value after it was released"
	}
}

proc check_exit_hold { job } {
	global scontrol number

	set matches 0
	spawn $scontrol -a --local show job $job
	expect {
		"Priority=0" {
			incr matches
			exp_continue
		}
		"Reason=JobHeldUser" {
			incr matches
			exp_continue
		}
		timeout {
			fail "scontrol is not responding"
		}
		eof {
			wait
		}
	}

	if { $matches != 2 } {
		fail "Priority was not set to a non zero value after it was released"
	}
}

proc check_restart_cnt { file cnt } {
	global bin_grep bin_sleep

	# wait for prolog, etc. to finish
	sleep 5

	set match 0
	spawn $bin_grep "SLURM_RESTART_COUNT" $file
	expect {
		"SLURM_RESTART_COUNT=$cnt" {
			set match 1
			exp_continue
		}
		timeout {
			fail "grep ($file) not responding"
		}
		eof {
			wait
		}
	}
	if {!$match} {
		fail "Failed to find SLURM_RESTART_COUNT=$cnt in output file"
	}
}

spawn $bin_id -un
expect {
	-re "($re_word_str)" {
		set user_name $expect_out(1,string)
	}
	eof {
		wait
	}
}

proc get_slurm_conf { cluster } {
	global scontrol

	log_user 1
	set conf ""
	spawn $scontrol -M$cluster show config
	expect {
		-re "SLURM_CONF\\s+=\\s+(\\S+)" {
			set conf $expect_out(1,string)
			exp_continue
		}
		timeout {
			fail "scontrol not responding"
		}
		eof {
			wait
		}
	}
	log_user 1

	return $conf
}


# Remove existing setup
cleanup

# Add clusters to federation
if [setup_federation $fed_name] {
	fail "Failed to setup federation"
}

set requeue_exit_num [get_config_param "RequeueExit"]
set requeue_exithold_num [get_config_param "RequeueExitHold"]

make_bash_script $long_script     "sleep 9000"
make_bash_script $complete_script "env; $bin_sleep 25"
make_bash_script $fail_script     "BadCommand"
make_bash_script $prolog_script    "exit 0"
make_bash_script $prologctl_script "exit 0"
make_bash_script $epilog_script    "exit 0"

if {$requeue_exit_num ne "(null)"} {
	make_bash_script $exit_script     "env; $bin_sleep 25; exit $requeue_exit_num"
} else {
	log_warn "Configure RequeueExit=# to test."
}
if {$requeue_exithold_num ne "(null)"} {
	make_bash_script $exithold_script "env; $bin_sleep 25; exit $requeue_exithold_num"
} else {
	log_warn "Configure RequeueExitHold=# to test."
}


# get number of nodes per cluster
set node_count [llength [get_nodes_by_state idle,alloc,comp "[default_partition] --local"]]

set origin_cluster [get_config_param "ClusterName"]
set all_cluster_list [list $fedc1 $fedc2 $fedc3]
set non_origin_cluster_list [lsearch -all -inline -not -exact $all_cluster_list $origin_cluster]
set non_origin_clusters [join $non_origin_cluster_list ","]
log_info "Origin: $origin_cluster non-origins: $non_origin_clusters"


log_info "################################################################"
log_info "Test requeue of a running job on origin cluster"
log_info "################################################################"

# Submit jobs that fill up fed2,fed3
spawn $sbatch -N$node_count --exclusive -o /dev/null -e /dev/null -t999999 --requeue -M$fedc2 $long_script
expect {
	-re "Submitted batch job ($number)" {
		set long_running_job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		fail "sbatch is not responding"
	}
	eof {
		wait
	}
}
set run_cluster [wait_for_fed_job $long_running_job_id RUNNING $fedc2]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

spawn $sbatch -N$node_count --exclusive -o /dev/null -e /dev/null -t999999 --requeue -M$fedc3 $long_script
expect {
	-re "Submitted batch job ($number)" {
		set long_running_job_id2 $expect_out(1,string)
		exp_continue
	}
	timeout {
		fail "sbatch is not responding"
	}
	eof {
		wait
	}
}
set run_cluster [wait_for_fed_job $long_running_job_id2 RUNNING $fedc3]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

spawn $sbatch -N$node_count --exclusive -o /dev/null -e /dev/null -t1 --requeue $complete_script
expect {
	-re "Submitted batch job ($number)" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		fail "sbatch is not responding"
	}
	eof {
		wait
	}
}

if { $job_id == 0 } {
	fail "sbatch did not submit job"
}

# Wait for the job to be in the running state
set run_cluster [wait_for_fed_job $job_id RUNNING $origin_cluster]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

# Requeue the job while it is running
requeue_job $job_id

set pend_cluster [wait_for_fed_job $job_id PENDING $fedc1]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc2]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc3]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}

# Check to see if the job state is PENDING after the requeue
# federation will requeue job on all clusters
check_dbd_states $job_id REQUEUED $run_cluster 1
check_ctl_state $job_id PENDING $fedc1
check_ctl_state $job_id PENDING $fedc2
check_ctl_state $job_id PENDING $fedc3

if {[cancel_job $job_id]} {
	fail "Unable to cancel job ($job_id)"
}

log_info "################################################################"
log_info "Test requeue of a completed job on origin cluster"
log_info "################################################################"

spawn $sbatch -N$node_count --exclusive -o /dev/null -e /dev/null -t1 --requeue $complete_script
expect {
	-re "Submitted batch job ($number)" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		fail "sbatch is not responding"
	}
	eof {
		wait
	}
}

if { $job_id == 0 } {
	fail "sbatch did not submit job"
}
set run_cluster [wait_for_fed_job $job_id RUNNING ""]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

# Wait for the job to be in the complete state
set done_cluster [wait_for_fed_job $job_id DONE $run_cluster]
if {$done_cluster eq ""} {
	fail "Didn't find cluster with completed job"
}

# Requeue the job when it is complete
requeue_job $job_id

set pend_cluster [wait_for_fed_job $job_id PENDING $fedc1]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc2]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc3]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}

# Check to see if the job state is PENDING after the requeue
# federation will requeue job on all clusters
check_dbd_states $job_id REQUEUED $run_cluster 1
check_ctl_state $job_id PENDING $fedc1
check_ctl_state $job_id PENDING $fedc2
check_ctl_state $job_id PENDING $fedc3

if {[cancel_job $job_id]} {
	fail "Unable to cancel job ($job_id)"
}

log_info "################################################################"
log_info "Test requeue of a failed job on origin cluster"
log_info "################################################################"
set job_id 0
spawn $sbatch -N1 -o /dev/null -e /dev/null -t 1 --requeue $fail_script
expect {
	-re "Submitted batch job ($number)" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		fail "sbatch is not responding"
	}
	eof {
		wait
	}
}

if { $job_id == 0 } {
	fail "sbatch did not submit job"
}

# Wait for the job to be in the complete state
set run_cluster [wait_for_fed_job $job_id DONE $origin_cluster]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

# Requeue the job when it is complete
requeue_job $job_id

set pend_cluster [wait_for_fed_job $job_id PENDING $fedc1]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc2]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc3]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}

# Check to see if the job state is PENDING after the requeue
# federation will requeue job on all clusters
check_dbd_states $job_id REQUEUED $run_cluster 1
check_ctl_state $job_id PENDING $fedc1
check_ctl_state $job_id PENDING $fedc2
check_ctl_state $job_id PENDING $fedc3

if {[cancel_job $job_id]} {
	fail "Unable to cancel job ($job_id)"
}
if {[cancel_job $long_running_job_id]} {
	fail "Unable to cancel job ($long_running_job_id)"
}
if {[cancel_job $long_running_job_id2]} {
	fail "Unable to cancel job ($long_running_job_id2)"
}

log_info "################################################################"
log_info "Test requeue of running job on sibling cluster"
log_info "################################################################"

# Submit job that consumes all nodes on first cluster
spawn $sbatch -N$node_count --exclusive -o /dev/null -e /dev/null -t999999 --requeue -M$origin_cluster $long_script
expect {
	-re "Submitted batch job ($number)" {
		set long_running_job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		fail "sbatch is not responding"
	}
	eof {
		wait
	}
}
set run_cluster [wait_for_fed_job $long_running_job_id RUNNING $origin_cluster]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

spawn $sbatch -N$node_count --exclusive -o /dev/null -e /dev/null -t1 --requeue $complete_script
expect {
	-re "Submitted batch job ($number)" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		fail "sbatch is not responding"
	}
	eof {
		wait
	}
}

if { $job_id == 0 } {
	fail "sbatch did not submit job"
}

set run_cluster [wait_for_fed_job $job_id RUNNING $non_origin_clusters]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

log_info "$run_cluster is running job"

# make sure that the origin has gotten word that the job is running and the
# origin revokes the job.
set rv_origin_cluster [wait_for_fed_job $job_id REVOKED $origin_cluster]
if {$rv_origin_cluster eq ""} {
	fail "Origin cluster hasn't revoked job"
}

# Requeue the job while it is running
requeue_job $job_id

set pend_cluster [wait_for_fed_job $job_id PENDING $fedc1]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc2]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc3]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}

# Give time for states to be updated in the db.
sleep $dbd_delay

# Check to see if the job state is PENDING after the requeue
# federation will requeue job on all clusters
check_dbd_states $job_id REVOKED $origin_cluster 1
check_dbd_states $job_id PENDING $origin_cluster 1
check_dbd_states $job_id REQUEUED $run_cluster 1
check_dbd_states $job_id PENDING  $run_cluster 1
check_ctl_state $job_id PENDING $fedc1
check_ctl_state $job_id PENDING $fedc2
check_ctl_state $job_id PENDING $fedc3

if {[cancel_job $job_id]} {
	fail "Unable to cancel job ($job_id)"
}

log_info "################################################################"
log_info "Test requeue on subset of siblings"
log_info "################################################################"

spawn $sbatch -N$node_count --exclusive -o /dev/null -e /dev/null -t1 --requeue -M$fedc1,$fedc2 $complete_script
expect {
	-re "Submitted batch job ($number)" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		fail "sbatch is not responding"
	}
	eof {
		wait
	}
}

if { $job_id == 0 } {
	fail "sbatch did not submit job"
}

set run_cluster [wait_for_fed_job $job_id RUNNING $non_origin_clusters]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

log_info "$run_cluster is running job"

# make sure that the origin has gotten word that the job is running and the
# origin revokes the job.
set rv_origin_cluster [wait_for_fed_job $job_id REVOKED $origin_cluster]
if {$rv_origin_cluster eq ""} {
	fail "Origin cluster hasn't revoked job"
}

# Requeue the job while it is running
requeue_job $job_id

set pend_cluster [wait_for_fed_job $job_id PENDING $fedc1]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc2]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}

# Give time for states to be updated in the db.
sleep $dbd_delay

# Check to see if the job state is PENDING after the requeue
# federation will requeue job on all clusters
check_dbd_states $job_id REVOKED $origin_cluster 1
check_dbd_states $job_id PENDING $origin_cluster 1
check_dbd_states $job_id REQUEUED $run_cluster 1
check_dbd_states $job_id PENDING  $run_cluster 1
check_ctl_state $job_id PENDING $fedc1
check_ctl_state $job_id PENDING $fedc2
check_missing_job $job_id $fedc3

if {[cancel_job $job_id]} {
	fail "Unable to cancel job ($job_id)"
}

log_info "################################################################"
log_info "Test requeue of a completed job that ran on sibling"
log_info "################################################################"

# long running job is already running on origin cluster so job should go to
# other cluster

spawn $sbatch -N$node_count --exclusive -o $file_out -e /dev/null -t1 --requeue $complete_script
expect {
	-re "Submitted batch job ($number)" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		fail "sbatch is not responding"
	}
	eof {
		wait
	}
}

if { $job_id == 0 } {
	fail "sbatch did not submit job"
}

set run_cluster [wait_for_fed_job $job_id RUNNING $non_origin_clusters]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

log_info "$run_cluster is running job"

set ret_cluster [wait_for_fed_job $job_id DONE $run_cluster]
if {$ret_cluster eq ""} {
	fail "Didn't find cluster with completed job"
}

# Requeue the job when it is complete
requeue_job $job_id

set pend_cluster [wait_for_fed_job $job_id PENDING $fedc1]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc2]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc3]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}

# Give time for states to be updated in the db.
sleep $dbd_delay

# Check to see if the job state is PENDING after the requeue
# federation will requeue job on all clusters
check_dbd_states $job_id REVOKED $origin_cluster 1
check_dbd_states $job_id PENDING $origin_cluster 1

# A completed job on a sibling could already be gone from the controller and
# the db_index could be lost so the dbd state will stay as completed.
#check_dbd_states $job_id REQUEUED $run_cluster 1
check_dbd_states $job_id COMPLETED $run_cluster 1

check_dbd_states $job_id PENDING  $run_cluster 1
check_ctl_state $job_id PENDING $fedc1
check_ctl_state $job_id PENDING $fedc2
check_ctl_state $job_id PENDING $fedc3

log_info "################################################################"
log_info "Test that SLURM_RESTART_COUNT is set for job requeued on sibling"
log_info "################################################################"

set run_cluster [wait_for_fed_job $job_id RUNNING $non_origin_clusters]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

if {$run_cluster eq $origin_cluster} {
	fail "Requeued job ran on origin cluster. Expected to run on a different cluster"
}

check_restart_cnt $file_out 1

# check that sibling that didn't run the job has a REVOKED state in the db.
set revoked_sib ""
if {$fedc1 ne $origin_cluster && $fedc1 ne $run_cluster} {
	    set revoked_sib $fedc1

} elseif {$fedc2 ne $origin_cluster && $fedc2 ne $run_cluster} {
	    set revoked_sib $fedc2

} else {
	    set revoked_sib $fedc3
}
check_dbd_states $job_id REVOKED $revoked_sib 1

if {[cancel_job $job_id]} {
	fail "Unable to cancel job ($job_id)"
}

log_info "################################################################"
log_info "Test requeue of a cancelled job that ran on sibling"
log_info "################################################################"

# long running job is already running on origin cluster so job should go to
# other cluster

spawn $sbatch -N$node_count --exclusive -o $file_out -e /dev/null -t1 --requeue $complete_script
expect {
	-re "Submitted batch job ($number)" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		fail "sbatch is not responding"
	}
	eof {
		wait
	}
}

if { $job_id == 0 } {
	fail "sbatch did not submit job"
}

set run_cluster [wait_for_fed_job $job_id RUNNING $non_origin_clusters]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}
# Give time for origin to know that the job was started. If the cancel comes
# before it knows that the job started then the origin will cancel the local
# and remote job. If it knows that the job is running on the remote, then it
# will only send the request to the remote and wait for it to report back that
# the job is gone.
sleep 2

log_info "$run_cluster is running job"

if {[cancel_job $job_id]} {
	fail "Unable to cancel job ($job_id)"
}

set ret_cluster [wait_for_fed_job $job_id DONE $run_cluster]
if {$ret_cluster eq ""} {
	fail "Didn't find cluster with completed job"
}

# Requeue the job when it is complete
requeue_job $job_id

set pend_cluster [wait_for_fed_job $job_id PENDING $fedc1]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc2]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc3]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}

# Give time for states to be updated in the db.
sleep $dbd_delay

check_dbd_states $job_id REVOKED $origin_cluster 1
check_dbd_states $job_id PENDING $origin_cluster 1
#check_dbd_states $job_id REQUEUED $run_cluster 1
check_dbd_states $job_id PENDING  $run_cluster 1
check_ctl_state $job_id PENDING $fedc1
check_ctl_state $job_id PENDING $fedc2
check_ctl_state $job_id PENDING $fedc3

set run_cluster [wait_for_fed_job $job_id RUNNING $non_origin_clusters]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

if {$run_cluster eq $origin_cluster} {
	fail "Requeued job ran on origin cluster. Expected to run on a different cluster"
}

check_restart_cnt $file_out 1

if {[cancel_job $job_id]} {
	fail "Unable to cancel job ($job_id)"
}


log_info "################################################################"
log_info "Test scontrol requeuehold,release of a running sibling job"
log_info "################################################################"

# long running job is already running on origin cluster so job should go to
# other cluster

spawn $sbatch -N$node_count --exclusive -o $file_out -e /dev/null -t1 --requeue $complete_script
expect {
	-re "Submitted batch job ($number)" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		fail "sbatch is not responding"
	}
	eof {
		wait
	}
}

if { $job_id == 0 } {
	fail "sbatch did not submit job"
}

set run_cluster [wait_for_fed_job $job_id RUNNING $non_origin_clusters]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

log_info "$run_cluster is running job"

# make sure that the origin has gotten word that the job is running and the
# origin revokes the job.
set rv_origin_cluster [wait_for_fed_job $job_id REVOKED $origin_cluster]
if {$rv_origin_cluster eq ""} {
	fail "Origin cluster hasn't revoked job"
}

# Requeue the job while it is running
requeuehold_job $job_id

set pend_cluster [wait_for_fed_job $job_id PENDING $origin_cluster]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

check_hold $job_id

# Give time for states to be updated in the db.
sleep $dbd_delay

check_ctl_state $job_id PENDING $origin_cluster
check_dbd_states $job_id REVOKED $origin_cluster 1
check_dbd_states $job_id PENDING $origin_cluster 1


# release hold on job, siblings will be submitted to all clusters
release_job $job_id

set pend_cluster [wait_for_fed_job $job_id PENDING $fedc1]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc2]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc3]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}

check_ctl_state $job_id PENDING $fedc1
check_ctl_state $job_id PENDING $fedc2
check_ctl_state $job_id PENDING $fedc3

set run_cluster [wait_for_fed_job $job_id RUNNING $non_origin_clusters]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

if {$run_cluster eq $origin_cluster} {
	fail "Requeued job ran on origin cluster. Expected to run on a different cluster"
}

check_restart_cnt $file_out 1

log_info "################################################################"
log_info "Test scontrol requeuehold,release of a completed sibling job"
log_info "################################################################"

# Just let previous job finish and then requeue it.

set run_cluster [wait_for_fed_job $job_id DONE $run_cluster]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with completed job"
}

# Requeue the job when it is complete
requeuehold_job $job_id

set run_cluster [wait_for_fed_job $job_id PENDING $origin_cluster]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

check_hold $job_id

# Give time for states to be updated in the db.
sleep $dbd_delay

check_ctl_state $job_id PENDING $origin_cluster
check_dbd_states $job_id REVOKED $origin_cluster 1
check_dbd_states $job_id PENDING $origin_cluster 1

# release hold on job, siblings will be submitted to all clusters
release_job $job_id

set pend_cluster [wait_for_fed_job $job_id PENDING $fedc1]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with running job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc2]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with running job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc3]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

check_ctl_state $job_id PENDING $fedc1
check_ctl_state $job_id PENDING $fedc2
check_ctl_state $job_id PENDING $fedc3

set run_cluster [wait_for_fed_job $job_id RUNNING $non_origin_clusters]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

if {$run_cluster eq $origin_cluster} {
	fail "Requeued job ran on origin cluster. Expected to run on a different cluster"
}

check_restart_cnt $file_out 2
if {[cancel_job $job_id]} {
	fail "Unable to cancel job ($job_id)"
}

log_info "################################################################"
log_info "Test scontrol requeuehold specialexit,release of a running sibling job"
log_info "################################################################"

# long running job is already running on origin cluster so job should go to
# other cluster

spawn $sbatch -N$node_count --exclusive -o $file_out -e /dev/null -t1 --requeue $complete_script
expect {
	-re "Submitted batch job ($number)" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		fail "sbatch is not responding"
	}
	eof {
		wait
	}
}

if { $job_id == 0 } {
	fail "sbatch did not submit job"
}

set run_cluster [wait_for_fed_job $job_id RUNNING $non_origin_clusters]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

log_info "$run_cluster is running job"

# make sure that the origin has gotten word that the job is running and the
# origin revokes the job.
set rv_origin_cluster [wait_for_fed_job $job_id REVOKED $origin_cluster]
if {$rv_origin_cluster eq ""} {
	fail "Origin cluster hasn't revoked job"
}

# Requeue the job when it is complete
requeuehold_se_job $job_id

set se_cluster [wait_for_fed_job $job_id SPECIAL_EXIT $origin_cluster]
if {$se_cluster eq ""} {
	fail "Didn't find job in SE state"
}
check_hold $job_id
check_ctl_state $job_id SPECIAL_EXIT $origin_cluster

# Give time for states to be updated in the db.
sleep $dbd_delay

check_dbd_states $job_id REVOKED $origin_cluster 1
check_dbd_states $job_id PENDING $origin_cluster 1


# release hold on job, siblings will be submitted to all clusters
release_job $job_id

set pend_cluster [wait_for_fed_job $job_id PENDING $fedc1]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc2]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc3]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}

check_ctl_state $job_id PENDING $fedc1
check_ctl_state $job_id PENDING $fedc2
check_ctl_state $job_id PENDING $fedc3

set run_cluster [wait_for_fed_job $job_id RUNNING $non_origin_clusters]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

if {$run_cluster eq $origin_cluster} {
	fail "Requeued job ran on origin cluster. Expected to run on a different cluster"
}

check_restart_cnt $file_out 1

log_info "################################################################"
log_info "Test scontrol requeuehold specialexit,release of a completed sibling job"
log_info "################################################################"

# Just let previous job finish and then requeue it.

set run_cluster [wait_for_fed_job $job_id DONE $run_cluster]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with completed job"
}

# Requeue the job when it is complete
requeuehold_se_job $job_id

set se_cluster [wait_for_fed_job $job_id SPECIAL_EXIT $origin_cluster]
if {$se_cluster eq ""} {
	fail "Didn't find job in SE state"
}
check_hold $job_id
check_ctl_state $job_id SPECIAL_EXIT $origin_cluster

# Give time for states to be updated in the db.
sleep $dbd_delay

check_dbd_states $job_id REVOKED $origin_cluster 1
check_dbd_states $job_id PENDING $origin_cluster 1


# release hold on job, siblings will be submitted to all clusters
release_job $job_id

set pend_cluster [wait_for_fed_job $job_id PENDING $fedc1]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc2]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}
set pend_cluster [wait_for_fed_job $job_id PENDING $fedc3]
if {$pend_cluster eq ""} {
	fail "Didn't find cluster with pending job"
}

check_ctl_state $job_id PENDING $fedc1
check_ctl_state $job_id PENDING $fedc2
check_ctl_state $job_id PENDING $fedc3

set run_cluster [wait_for_fed_job $job_id RUNNING $non_origin_clusters]
if {$run_cluster eq ""} {
	fail "Didn't find cluster with running job"
}

if {$run_cluster eq $origin_cluster} {
	fail "Requeued job ran on origin cluster. Expected to run on a different cluster"
}

check_restart_cnt $file_out 2
if {[cancel_job $job_id]} {
	fail "Unable to cancel job ($job_id)"
}

if {$requeue_exit_num ne "(null)"} {
	log_info "################################################################"
	log_info "Test RequeueExit=#"
	log_info "################################################################"

	spawn $sbatch -N$node_count --exclusive -o $file_out -e /dev/null -t1 --requeue $exit_script
	expect {
		-re "Submitted batch job ($number)" {
			set job_id $expect_out(1,string)
			exp_continue
		}
		timeout {
			fail "sbatch is not responding"
		}
		eof {
			wait
		}
	}

	if { $job_id == 0 } {
		fail "sbatch did not submit job"
	}

	set run_cluster [wait_for_fed_job $job_id RUNNING $non_origin_clusters]
	if {$run_cluster eq ""} {
		fail "Didn't find cluster with running job"
	}

	if {$run_cluster eq $origin_cluster} {
		fail "Requeued job ran on origin cluster. Expected to run on a different cluster"
	}

	log_info "$run_cluster is running job"

	set pend_cluster [wait_for_fed_job $job_id PENDING $fedc1]
	if {$pend_cluster eq ""} {
		fail "Didn't find job in pending state"
	}
	set pend_cluster [wait_for_fed_job $job_id PENDING $fedc2]
	if {$pend_cluster eq ""} {
		fail "Didn't find job in pending state"
	}
	set pend_cluster [wait_for_fed_job $job_id PENDING $fedc3]
	if {$pend_cluster eq ""} {
		fail "Didn't find job in pending state"
	}

	# Give time for states to be updated in the db.
	sleep $dbd_delay
	check_dbd_states $job_id REQUEUED $origin_cluster 1
	check_dbd_states $job_id PENDING $origin_cluster 1
	#check_dbd_states $job_id REVOKED $run_cluster 1
	check_ctl_state $job_id PENDING $fedc1
	check_ctl_state $job_id PENDING $fedc2
	check_ctl_state $job_id PENDING $fedc3

	set run_cluster [wait_for_fed_job $job_id RUNNING $non_origin_clusters]
	if {$run_cluster eq ""} {
		fail "Didn't find cluster with running job"
	}

	check_restart_cnt $file_out 1
	if {[cancel_job $job_id]} {
		fail "Unable to cancel job ($job_id)"
	}
}

if {$requeue_exit_num ne "(null)"} {
	log_info "################################################################"
	log_info "Test RequeueExitHold=#"
	log_info "################################################################"

	spawn $sbatch -N$node_count --exclusive -o $file_out -e /dev/null -t1 --requeue $exithold_script
	expect {
		-re "Submitted batch job ($number)" {
			set job_id $expect_out(1,string)
			exp_continue
		}
		timeout {
			fail "sbatch is not responding"
		}
		eof {
			wait
		}
	}

	if { $job_id == 0 } {
		fail "sbatch did not submit job"
	}

	set run_cluster [wait_for_fed_job $job_id RUNNING $non_origin_clusters]
	if {$run_cluster eq ""} {
		fail "Didn't find cluster with running job"
	}

	if {$run_cluster eq $origin_cluster} {
		fail "Requeued job ran on origin cluster. Expected to run on a different cluster"
	}

	log_info "$run_cluster is running job"

	set pend_cluster [wait_for_fed_job $job_id SPECIAL_EXIT $origin_cluster]
	if {$pend_cluster eq ""} {
		fail "Didn't find job in pending state"
	}

	check_exit_hold $job_id
	# Give time for states to be updated in the db.
	sleep $dbd_delay

	check_ctl_state $job_id SPECIAL_EXIT $origin_cluster
	check_dbd_states $job_id REQUEUED $origin_cluster 1
	check_dbd_states $job_id PENDING $origin_cluster 1

	# release hold on job, siblings will be submitted to all clusters
	release_job $job_id

	set pend_cluster [wait_for_fed_job $job_id PENDING $fedc1]
	if {$pend_cluster eq ""} {
		fail "Didn't find cluster with pending job"
	}
	set pend_cluster [wait_for_fed_job $job_id PENDING $fedc2]
	if {$pend_cluster eq ""} {
		fail "Didn't find cluster with pending job"
	}
	set pend_cluster [wait_for_fed_job $job_id PENDING $fedc3]
	if {$pend_cluster eq ""} {
		fail "Didn't find cluster with pending job"
	}

	check_ctl_state $job_id PENDING $fedc1
	check_ctl_state $job_id PENDING $fedc2
	check_ctl_state $job_id PENDING $fedc3

	set run_cluster [wait_for_fed_job $job_id RUNNING $non_origin_clusters]
	if {$run_cluster eq ""} {
		fail "Didn't find cluster with running job"
	}

	if {$run_cluster eq $origin_cluster} {
		fail "Requeued job ran on origin cluster. Expected to run on a different cluster"
	}

	check_restart_cnt $file_out 1

	if {[cancel_job $job_id]} {
		fail "Unable to cancel job ($job_id)"
	}
}