testsuite/expect/test6.15 - SchedMD/slurm - Git at Google

 #!/usr/bin/env expect
 ############################################################################
 # Purpose: Test scancel with --full and/or --batch.
 #
 # Note:    This script generates and then deletes files in the working directory
 #          named test6.15.input, test6.15.output, and test6.15.error
 ############################################################################
 # Copyright (C) SchedMD LLC.
 #
 # This file is part of Slurm, a resource management program.
 # For details, see <https://slurm.schedmd.com/>.
 # Please also read the included file: DISCLAIMER.
 #
 # Slurm is free software; you can redistribute it and/or modify it under
 # the terms of the GNU General Public License as published by the Free
 # Software Foundation; either version 2 of the License, or (at your option)
 # any later version.
 #
 # Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 # details.
 #
 # You should have received a copy of the GNU General Public License along
 # with Slurm; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
 ############################################################################
 source ./globals

 set file_in     "$test_dir/input"
 set file_out    "$test_dir/output"
 set file_prog   "$test_name.prog"
 set job_id      0

 proc cleanup {} {
 	global job_id file_prog

 	cancel_job $job_id
 	file delete $file_prog
 }

 # Build prog
 run_command -fail "$bin_cc -o $file_prog $file_prog.c"

 #
 # Build input script files
 #
 make_bash_script $file_in "
 catch()
 {
 	echo 'Signaled: script'
 	exit 0
 }

 trap catch USR1

 $bin_echo Job \$SLURM_JOB_ID is running
 ./test6.15.prog &
 $srun ./test6.15.prog step
 wait
 exit 0
 "

 set proctracktype [get_config_param "ProctrackType"]
 log_debug "Proctrack: $proctracktype"
 if {$proctracktype ne "proctrack/cgroup" && $proctracktype ne "proctrack/linuxproc" && $proctracktype ne "proctrack/pgid"} {
 	skip "This test only works for proctrack/cgroup, proctrack/linuxproc or proctrack/pgid"
 }

 #
 # Main testproc function
 #
 # It submits a batch script that runs a background command and a step.
 # Both command and step are the same app that creates parent and child process.
 # All three, the script, the command and the step have a signal handler
 # for SIGUSR1 to print that the signal is received.
 #
 # The function uses scancel to signal the job, and finally check the stdout
 # to verify that the signals are received as expected.
 # The scancel can be used with $params to check them.
 #
 proc test_signaling { batch full } {
 	global max_job_delay proctracktype
 	global bin_sleep bin_cat job_id
 	global sbatch scancel file_out file_in
 	global number re_word_str bin_grep

 	# Cleanup result from previous call
 	file delete $file_out

 	#
 	# Submit sbatch job
 	#
 	log_debug "Submit the job"
 	set timeout $max_job_delay
 	set job_id [submit_job -fail "--output=$file_out -e /dev/null -t2 $file_in"]

 	#
 	# Wait for job to start running
 	#
 	# If slurmd gets the job signal request before the tasks have actually been
 	# started, the asynchronous call can fail to signal the job script. Wait
 	# for evidence that the job has started and has written to its output file.
 	wait_for_job -fail $job_id "RUNNING"
 	wait_for_file -fail $file_out
 	wait_for_command_match -fail -pollinterval .1 "$bin_grep -c Started $file_out" 4

 	#
 	# Signal the job
 	#
 	set params ""
 	if { $batch } {
 		set params [concat $params " --batch"]
 	}
 	if { $full } {
 		set params [concat $params " --full"]
 	}

 	log_debug "Signal the job"
 	run_command -fail "$scancel -s SIGUSR1 $params $job_id"

 	#
 	# Wait for job completion
 	#
 	wait_for_job -fail $job_id DONE

 	set expected_script_signaled 0
 	set expected_parent_step_signaled 0
 	set expected_parent_command_signaled 0
 	set expected_child_step_signaled 0
 	set expected_child_command_signaled 0

 	if {$batch && !$full} {
 		set expected_script_signaled 1
 	} elseif {!$batch && !$full} {
 		set expected_parent_step_signaled 1
 		if {$proctracktype == "proctrack/linuxproc" || $proctracktype == "proctrack/pgid"} {
 			set expected_child_step_signaled 1
 		}
 	} else {
 		set expected_script_signaled 1
 		set expected_parent_step_signaled 1
 		set expected_parent_command_signaled 1
 		set expected_child_command_signaled 1
 		if {$proctracktype == "proctrack/linuxproc" || $proctracktype == "proctrack/pgid"} {
 			set expected_child_step_signaled 1
 		}
 	}

 	#
 	# Grep the output file for the signaled process
 	#
 	set script_signaled          0
 	set parent_step_signaled     0
 	set parent_command_signaled  0
 	set child_step_signaled      0
 	set child_command_signaled   0

 	spawn $bin_cat $file_out
 	expect {
 		-re "Signaled: ($re_word_str)" {
 			set signaled $expect_out(1,string)
 			if {$signaled == "script"} {
 				incr script_signaled
 			} elseif {$signaled == "parent_step"} {
 				incr parent_step_signaled
 			} elseif {$signaled == "parent_command"} {
 				incr parent_command_signaled
 			} elseif {$signaled == "child_step"} {
 				incr child_step_signaled
 			} elseif {$signaled == "child_command"} {
 				incr child_command_signaled
 			}
 			exp_continue
 		}
 		eof {
 			wait
 		}
 	}
 	subtest {$script_signaled == $expected_script_signaled} "Script should [expr ($expected_script_signaled==1)?{be}:{NOT be}] signaled"
 	subtest {$parent_step_signaled == $expected_parent_step_signaled} "Parent step should [expr ($expected_parent_step_signaled==1)?{be}:{NOT be}] signaled"
 	subtest {$parent_command_signaled == $expected_parent_command_signaled} "Parent command should [expr ($expected_parent_command_signaled==1)?{be}:{NOT be}] signaled"
 	subtest {$child_step_signaled == $expected_child_step_signaled} "Child step should [expr ($expected_child_step_signaled==1)?{be}:{NOT be}] signaled"
 	subtest {$child_command_signaled == $expected_child_command_signaled} "Child command should [expr ($expected_child_command_signaled==1)?{be}:{NOT be}] signaled"
 }

 #
 # Run the tests
 #
 testproc test_signaling 0 0
 testproc test_signaling 1 0
 testproc test_signaling 0 1
 testproc test_signaling 1 1
	#!/usr/bin/env expect
	############################################################################
	# Purpose: Test scancel with --full and/or --batch.
	#
	# Note: This script generates and then deletes files in the working directory
	# named test6.15.input, test6.15.output, and test6.15.error
	############################################################################
	# Copyright (C) SchedMD LLC.
	#
	# This file is part of Slurm, a resource management program.
	# For details, see <https://slurm.schedmd.com/>.
	# Please also read the included file: DISCLAIMER.
	#
	# Slurm is free software; you can redistribute it and/or modify it under
	# the terms of the GNU General Public License as published by the Free
	# Software Foundation; either version 2 of the License, or (at your option)
	# any later version.
	#
	# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
	# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
	# details.
	#
	# You should have received a copy of the GNU General Public License along
	# with Slurm; if not, write to the Free Software Foundation, Inc.,
	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	############################################################################
	source ./globals

	set file_in "$test_dir/input"
	set file_out "$test_dir/output"
	set file_prog "$test_name.prog"
	set job_id 0

	proc cleanup {} {
	global job_id file_prog

	cancel_job $job_id
	file delete $file_prog
	}

	# Build prog
	run_command -fail "$bin_cc -o $file_prog $file_prog.c"

	#
	# Build input script files
	#
	make_bash_script $file_in "
	catch()
	{
	echo 'Signaled: script'
	exit 0
	}

	trap catch USR1

	$bin_echo Job \$SLURM_JOB_ID is running
	./test6.15.prog &
	$srun ./test6.15.prog step
	wait
	exit 0
	"

	set proctracktype [get_config_param "ProctrackType"]
	log_debug "Proctrack: $proctracktype"
	if {$proctracktype ne "proctrack/cgroup" && $proctracktype ne "proctrack/linuxproc" && $proctracktype ne "proctrack/pgid"} {
	skip "This test only works for proctrack/cgroup, proctrack/linuxproc or proctrack/pgid"
	}

	#
	# Main testproc function
	#
	# It submits a batch script that runs a background command and a step.
	# Both command and step are the same app that creates parent and child process.
	# All three, the script, the command and the step have a signal handler
	# for SIGUSR1 to print that the signal is received.
	#
	# The function uses scancel to signal the job, and finally check the stdout
	# to verify that the signals are received as expected.
	# The scancel can be used with $params to check them.
	#
	proc test_signaling { batch full } {
	global max_job_delay proctracktype
	global bin_sleep bin_cat job_id
	global sbatch scancel file_out file_in
	global number re_word_str bin_grep

	# Cleanup result from previous call
	file delete $file_out

	#
	# Submit sbatch job
	#
	log_debug "Submit the job"
	set timeout $max_job_delay
	set job_id [submit_job -fail "--output=$file_out -e /dev/null -t2 $file_in"]

	#
	# Wait for job to start running
	#
	# If slurmd gets the job signal request before the tasks have actually been
	# started, the asynchronous call can fail to signal the job script. Wait
	# for evidence that the job has started and has written to its output file.
	wait_for_job -fail $job_id "RUNNING"
	wait_for_file -fail $file_out
	wait_for_command_match -fail -pollinterval .1 "$bin_grep -c Started $file_out" 4

	#
	# Signal the job
	#
	set params ""
	if { $batch } {
	set params [concat $params " --batch"]
	}
	if { $full } {
	set params [concat $params " --full"]
	}

	log_debug "Signal the job"
	run_command -fail "$scancel -s SIGUSR1 $params $job_id"

	#
	# Wait for job completion
	#
	wait_for_job -fail $job_id DONE

	set expected_script_signaled 0
	set expected_parent_step_signaled 0
	set expected_parent_command_signaled 0
	set expected_child_step_signaled 0
	set expected_child_command_signaled 0

	if {$batch && !$full} {
	set expected_script_signaled 1
	} elseif {!$batch && !$full} {
	set expected_parent_step_signaled 1
	if {$proctracktype == "proctrack/linuxproc" \|\| $proctracktype == "proctrack/pgid"} {
	set expected_child_step_signaled 1
	}
	} else {
	set expected_script_signaled 1
	set expected_parent_step_signaled 1
	set expected_parent_command_signaled 1
	set expected_child_command_signaled 1
	if {$proctracktype == "proctrack/linuxproc" \|\| $proctracktype == "proctrack/pgid"} {
	set expected_child_step_signaled 1
	}
	}

	#
	# Grep the output file for the signaled process
	#
	set script_signaled 0
	set parent_step_signaled 0
	set parent_command_signaled 0
	set child_step_signaled 0
	set child_command_signaled 0

	spawn $bin_cat $file_out
	expect {
	-re "Signaled: ($re_word_str)" {
	set signaled $expect_out(1,string)
	if {$signaled == "script"} {
	incr script_signaled
	} elseif {$signaled == "parent_step"} {
	incr parent_step_signaled
	} elseif {$signaled == "parent_command"} {
	incr parent_command_signaled
	} elseif {$signaled == "child_step"} {
	incr child_step_signaled
	} elseif {$signaled == "child_command"} {
	incr child_command_signaled
	}
	exp_continue
	}
	eof {
	wait
	}
	}
	subtest {$script_signaled == $expected_script_signaled} "Script should [expr ($expected_script_signaled==1)?{be}:{NOT be}] signaled"
	subtest {$parent_step_signaled == $expected_parent_step_signaled} "Parent step should [expr ($expected_parent_step_signaled==1)?{be}:{NOT be}] signaled"
	subtest {$parent_command_signaled == $expected_parent_command_signaled} "Parent command should [expr ($expected_parent_command_signaled==1)?{be}:{NOT be}] signaled"
	subtest {$child_step_signaled == $expected_child_step_signaled} "Child step should [expr ($expected_child_step_signaled==1)?{be}:{NOT be}] signaled"
	subtest {$child_command_signaled == $expected_child_command_signaled} "Child command should [expr ($expected_child_command_signaled==1)?{be}:{NOT be}] signaled"
	}

	#
	# Run the tests
	#
	testproc test_signaling 0 0
	testproc test_signaling 1 0
	testproc test_signaling 0 1
	testproc test_signaling 1 1