testsuite/expect/test7.11 - SchedMD/slurm - Git at Google

 #!/usr/bin/env expect
 ############################################################################
 # Purpose: Test of Slurm functionality
 #          Test of SPANK plugin
 ############################################################################
 # Copyright (C) 2008-2009 Lawrence Livermore National Security.
 # Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
 # Written by Morris Jette <jette1@llnl.gov>
 # CODE-OCEC-09-009. All rights reserved.
 #
 # This file is part of Slurm, a resource management program.
 # For details, see <https://slurm.schedmd.com/>.
 # Please also read the included file: DISCLAIMER.
 #
 # Slurm is free software; you can redistribute it and/or modify it under
 # the terms of the GNU General Public License as published by the Free
 # Software Foundation; either version 2 of the License, or (at your option)
 # any later version.
 #
 # Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 # details.
 #
 # You should have received a copy of the GNU General Public License along
 # with Slurm; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
 ############################################################################
 source ./globals

 set cwd             "[$bin_pwd]"
 set file_in         "$test_dir/input"
 set file_out        "$test_dir/output"
 set file_prog       "$cwd/$test_name.prog"
 set orig_spank_conf "$test_dir/orig_conf"
 set new_spank_conf  "$test_dir/new_conf"
 set spank_out       "$test_dir/spank.out"
 set job_id          0

 if {![is_super_user]} {
 	skip "This test must be run as SlurmUser"
 }

 set config_dir [get_conf_path]
 set spank_conf_file ${config_dir}/plugstack.conf

 regexp {^(\d+\.\d+)} [get_config_param SLURM_VERSION] - ctld_slurm_ver
 regexp {slurm (\d+\.\d+)} [run_command_output -fail "$scontrol -V"] - loc_slurm_ver
 if {$ctld_slurm_ver ne $loc_slurm_ver} {
 	skip "Slurmctld ($ctld_slurm_ver) and local Slurm ($loc_slurm_ver) versions are not the same, can not continue"
 }

 proc cleanup {} {
 	global job_id orig_spank_conf spank_conf_file
 	global bin_cp bin_rm file_prog test_status STATUS_FAIL

 	cancel_job $job_id

 	#
 	# Restore the original plugstack
 	#
 	if {[file exists $orig_spank_conf]} {
 		exec $bin_cp -f $orig_spank_conf $spank_conf_file
 	} else {
 		exec $bin_rm -f $spank_conf_file
 	}
 	reconfigure

 	file delete ${file_prog}.so

 	if {$test_status == $STATUS_FAIL} {
 		log_warn "Check for matching checksums on the plugstack.conf file. Different checksums could indicate file system delays"
 	}
 }

 #
 # Build the plugin
 #
 exec $bin_cc -fPIC -shared -I${slurm_dir}/include -o ${file_prog}.so ${file_prog}.c

 if {[file exists $spank_conf_file]} {
 	spawn $bin_cat $spank_conf_file
 	expect {
 		-re "test${test_id}" {
 			fail "Spank plugin includes vestigial test${test_id}. You probably should manually remove it from $spank_conf_file. It was probably left over from some previous test failure"
 		}
 		eof {
 			wait
 		}
 	}

 	exec $bin_cp -f $spank_conf_file $orig_spank_conf
 	exec $bin_cp -f $spank_conf_file $new_spank_conf
 } else {
 	exec $bin_cp -f /dev/null $new_spank_conf
 }

 exec $bin_echo "required ${file_prog}.so ${spank_out}" >>$new_spank_conf
 spawn $bin_cp -f $new_spank_conf $spank_conf_file
 expect {
 	-re "Permission denied" {
 		skip "User lacks permission to update plugstack_conf file"
 	}
 	eof {
 		wait
 	}
 }
 reconfigure -fail

 #
 # Test of srun help message
 #
 # NOTE: Expect parsing failures have been noted running "$srun --help"
 #       directly, so we build a script containing a pipe to tail
 #

 log_info "Testing srun help message........."

 make_bash_script $file_in "$srun --help | $bin_grep test"
 set matches 0
 spawn $file_in
 expect {
 	-re "Component of slurm test suite" {
 		incr matches
 		exp_continue
 	}
 	-re "Registered component of slurm test suite" {
 		incr matches
 		exp_continue
 	}
 	timeout {
 		fail "srun not responding"
 	}
 	eof {
 		wait
 	}
 }
 if {$matches != 2} {
 	fail "Spank help message not in srun help message"
 }

 #
 # Test of salloc help message
 #
 # NOTE: Expect parsing failures have been noted running "$salloc --help"
 #       directly, so we build a script containing a pipe to tail
 #

 log_info "Testing salloc help message........."

 make_bash_script $file_in "$salloc --help | $bin_grep test"
 set matches 0
 spawn $file_in
 expect {
 	-re "Registered component of slurm test suite" {
 		incr matches
 		exp_continue
 	}
 	timeout {
 		fail "salloc not responding"
 	}
 	eof {
 		wait
 	}
 }
 if {$matches != 1} {
 	fail "Spank help message not in salloc help message ($matches != 1)"
 }

 #
 # Test of sbatch help message
 #
 # NOTE: Expect parsing failures have been noted running "$sbatch --help"
 #       directly, so we build a script containing a pipe to tail
 #

 log_info "Testing sbatch help message........."

 make_bash_script $file_in "$sbatch --help | $bin_grep test"
 set matches 0
 spawn $file_in
 expect {
 	-re "Registered component of slurm test suite" {
 		incr matches
 		exp_continue
 	}
 	timeout {
 		fail "srun not responding"
 	}
 	eof {
 		wait
 	}
 }
 if {$matches != 1} {
 	fail "Spank help message not in sbatch help message ($matches != 1)"
 }

 #
 # Test of locally logged messages().
 # Add sleep for NFS delays in propagating $spank_conf_file
 #
 # NOTE: This test will fail if plugstack.conf is not in a shared location
 #	The login node (where test is started) will modify the file and the
 #	slurmd/slurmstepd on the compute node will not get the updated
 #	configuration.
 #
 log_info "Test locally logged messages........."

 log_debug "Checksum of local $spank_conf_file"
 spawn $bin_sum $spank_conf_file
 expect {
 	-re timeout {
 		fail "sbatch not responding"
 	}
 	eof {
 		wait
 	}
 }

 sleep 30
 make_bash_script $file_in "
 $srun --test_suite_srun=5 $bin_sum $spank_conf_file
 "

 set matches 0
 spawn $sbatch --test_suite_sbatch=4 -N1 -t1 -o $file_out $file_in
 expect {
 	-re "_test_opt_process_sbatch: opt_arg_sbatch=4" {
 		incr matches
 		exp_continue
 	}
 	-re "Submitted batch job ($number)" {
 		set job_id $expect_out(1,string)
 		exp_continue
 	}
 	-re "slurm_spank_exit: opt_arg_sbatch=4 opt_arg_srun=0" {
 		incr matches
 		exp_continue
 	}
 	-re timeout {
 		fail "sbatch not responding"
 	}
 	eof {
 		wait
 	}
 }
 if {$job_id == 0} {
 	fail "Batch submit failure"
 }
 if {$matches != 2} {
 	fail "Spank options not processed by sbatch ($matches != 2)"
 }

 #
 # Wait for job to complete and check for files
 #
 wait_for_job -fail $job_id "DONE"

 # NOTE: spank logs from sbatch and srun would be intermingled here
 wait_for_file -fail $file_out
 set matches 0
 set matches_sbatch 0
 set matches_srun 0
 spawn $bin_cat $file_out
 expect {
 	-re "error" {
 		fail "Some error happened"
 	}
 	-re "_test_opt_process_(.*?=$number)" {
 		set substr $expect_out(1,string)
 		if {[string equal "sbatch: opt_arg_sbatch=4" $substr]} {
 			incr matches_sbatch
 		} elseif {[string equal "srun: opt_arg_srun=5" $substr]} {
 			incr matches_srun
 		}
 		exp_continue
 	}
 	-re "slurm_spank_local_user_init" {
 		incr matches
 		exp_continue
 	}
 	-re "slurm_spank_exit: opt_arg_sbatch=4 opt_arg_srun=5" {
 		incr matches
 		exp_continue
 	}
 	eof {
 		wait
 	}
 }
 if {$matches_sbatch != 1} {
 	fail "Local (srun) sbatch spank plugin failure ($matches_sbatch != 1)"
 } elseif {$matches_srun != 1} {
 	fail "Local (srun) srun spank plugin failure ($matches_srun != 1)"
 } elseif {$matches != 2} {
 	fail "Local (srun) spank plugin failure ($matches != 2)"
 } else {
 	log_debug "Local (srun) spank plugin success"
 }

 #
 # Test for slurmd output in file
 #
 wait_for_file -fail $spank_out
 set matches        0
 set matches_sbatch 0
 set matches_srun   0
 set matches_spank_exit 0
 set matches_spank_init 0

 set expected_spank_exit 2
 set expected_spank_init 2
 set expected_sbatch     2
 set expected_srun       2
 set expected_items      5

 # If PrologFlags=contain is in slurm.conf, then an "extern" step will be
 # launched on each node, so we need to check for additional output
 log_user 0
 spawn $scontrol show config
 expect {
 	-re "PrologFlags\\s*=\\s*\[A-z/,\]*Contain" {
 		incr expected_spank_exit
 	}
 	timeout {
 		fail "scontrol show config not responding"
 	}
 	eof {
 		wait
 	}
 }
 log_user 1

 spawn $bin_sort $spank_out
 expect {
 	-re "slurm_spank_(\\S+): opt_arg_sbatch=($number) opt_arg_srun=($number)" {
 		set spank_type $expect_out(1,string)
 		set sbatch_arg $expect_out(2,string)
 		set srun_arg   $expect_out(3,string)

 		if {$spank_type eq "exit"} {
 			incr matches_spank_exit
 		} elseif {$spank_type eq "task_init"} {
 			incr matches_spank_init
 		}

 		if {($sbatch_arg eq "4") && ($srun_arg eq "0")} {
 			incr matches_sbatch
 		} elseif {($sbatch_arg eq "4") && ($srun_arg eq "5")} {
 			incr matches_srun
 		}
 		exp_continue
 	}
 	-re "spank_get_item: argv" {
 		incr matches
 		exp_continue
 	}
 	-re "spank_get_item: my_uid=" {
 		incr matches
 		exp_continue
 	}
 	eof {
 		wait
 	}
 }

 if {$matches_spank_exit != $expected_spank_exit} {
 	fail "Remote (slurmd) spank_exit spank plugin failure ($matches_spank_exit != $expected_spank_exit)"
 }
 if {$matches_spank_init != $expected_spank_init} {
 	fail "Remote (slurmd) spank_init spank plugin failure ($matches_spank_init != $expected_spank_init)"
 }
 if {$matches_sbatch != $expected_sbatch} {
 	fail "Remote (slurmd) sbatch spank plugin failure ($matches_sbatch != $expected_sbatch)"
 }
 if {$matches_srun != $expected_srun} {
 	fail "Remote (slurmd) srun spank plugin failure ($matches_srun != $expected_srun)"
 }
 if {$matches != $expected_items} {
 	fail "Remote (slurmd) spank plugin failure ($matches != $expected_items)"
 }
 log_debug "Remote (slurmd) spank plugin success"

 # Test if internal spank option parsing works (Bug 10454)
 set output [run_command_output -timeout 2 -xfail -subtest "$sbatch --test_suite_sbatch=NaN --wrap='/bin/true'"]
 subtest {[regexp "sbatch: error: Non-zero argument required" $output]} "sbatch should print error message from spank"

 set output [run_command_output -timeout 2 -xfail -subtest "$salloc --test_suite_sbatch=NaN"]
 subtest {[regexp "salloc: error: Non-zero argument required" $output]} "salloc should print error message from spank"

 set output [run_command_output -timeout 2 -xfail -subtest "$srun --test_suite_sbatch=NaN"]
 subtest {[regexp "srun: error: Non-zero argument required" $output]} "srun should print error message from spank"
	#!/usr/bin/env expect
	############################################################################
	# Purpose: Test of Slurm functionality
	# Test of SPANK plugin
	############################################################################
	# Copyright (C) 2008-2009 Lawrence Livermore National Security.
	# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
	# Written by Morris Jette <jette1@llnl.gov>
	# CODE-OCEC-09-009. All rights reserved.
	#
	# This file is part of Slurm, a resource management program.
	# For details, see <https://slurm.schedmd.com/>.
	# Please also read the included file: DISCLAIMER.
	#
	# Slurm is free software; you can redistribute it and/or modify it under
	# the terms of the GNU General Public License as published by the Free
	# Software Foundation; either version 2 of the License, or (at your option)
	# any later version.
	#
	# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
	# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
	# details.
	#
	# You should have received a copy of the GNU General Public License along
	# with Slurm; if not, write to the Free Software Foundation, Inc.,
	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	############################################################################
	source ./globals

	set cwd "[$bin_pwd]"
	set file_in "$test_dir/input"
	set file_out "$test_dir/output"
	set file_prog "$cwd/$test_name.prog"
	set orig_spank_conf "$test_dir/orig_conf"
	set new_spank_conf "$test_dir/new_conf"
	set spank_out "$test_dir/spank.out"
	set job_id 0

	if {![is_super_user]} {
	skip "This test must be run as SlurmUser"
	}

	set config_dir [get_conf_path]
	set spank_conf_file ${config_dir}/plugstack.conf

	regexp {^(\d+\.\d+)} [get_config_param SLURM_VERSION] - ctld_slurm_ver
	regexp {slurm (\d+\.\d+)} [run_command_output -fail "$scontrol -V"] - loc_slurm_ver
	if {$ctld_slurm_ver ne $loc_slurm_ver} {
	skip "Slurmctld ($ctld_slurm_ver) and local Slurm ($loc_slurm_ver) versions are not the same, can not continue"
	}

	proc cleanup {} {
	global job_id orig_spank_conf spank_conf_file
	global bin_cp bin_rm file_prog test_status STATUS_FAIL

	cancel_job $job_id

	#
	# Restore the original plugstack
	#
	if {[file exists $orig_spank_conf]} {
	exec $bin_cp -f $orig_spank_conf $spank_conf_file
	} else {
	exec $bin_rm -f $spank_conf_file
	}
	reconfigure

	file delete ${file_prog}.so

	if {$test_status == $STATUS_FAIL} {
	log_warn "Check for matching checksums on the plugstack.conf file. Different checksums could indicate file system delays"
	}
	}

	#
	# Build the plugin
	#
	exec $bin_cc -fPIC -shared -I${slurm_dir}/include -o ${file_prog}.so ${file_prog}.c

	if {[file exists $spank_conf_file]} {
	spawn $bin_cat $spank_conf_file
	expect {
	-re "test${test_id}" {
	fail "Spank plugin includes vestigial test${test_id}. You probably should manually remove it from $spank_conf_file. It was probably left over from some previous test failure"
	}
	eof {
	wait
	}
	}

	exec $bin_cp -f $spank_conf_file $orig_spank_conf
	exec $bin_cp -f $spank_conf_file $new_spank_conf
	} else {
	exec $bin_cp -f /dev/null $new_spank_conf
	}

	exec $bin_echo "required ${file_prog}.so ${spank_out}" >>$new_spank_conf
	spawn $bin_cp -f $new_spank_conf $spank_conf_file
	expect {
	-re "Permission denied" {
	skip "User lacks permission to update plugstack_conf file"
	}
	eof {
	wait
	}
	}
	reconfigure -fail

	#
	# Test of srun help message
	#
	# NOTE: Expect parsing failures have been noted running "$srun --help"
	# directly, so we build a script containing a pipe to tail
	#

	log_info "Testing srun help message........."

	make_bash_script $file_in "$srun --help \| $bin_grep test"
	set matches 0
	spawn $file_in
	expect {
	-re "Component of slurm test suite" {
	incr matches
	exp_continue
	}
	-re "Registered component of slurm test suite" {
	incr matches
	exp_continue
	}
	timeout {
	fail "srun not responding"
	}
	eof {
	wait
	}
	}
	if {$matches != 2} {
	fail "Spank help message not in srun help message"
	}

	#
	# Test of salloc help message
	#
	# NOTE: Expect parsing failures have been noted running "$salloc --help"
	# directly, so we build a script containing a pipe to tail
	#

	log_info "Testing salloc help message........."

	make_bash_script $file_in "$salloc --help \| $bin_grep test"
	set matches 0
	spawn $file_in
	expect {
	-re "Registered component of slurm test suite" {
	incr matches
	exp_continue
	}
	timeout {
	fail "salloc not responding"
	}
	eof {
	wait
	}
	}
	if {$matches != 1} {
	fail "Spank help message not in salloc help message ($matches != 1)"
	}

	#
	# Test of sbatch help message
	#
	# NOTE: Expect parsing failures have been noted running "$sbatch --help"
	# directly, so we build a script containing a pipe to tail
	#

	log_info "Testing sbatch help message........."

	make_bash_script $file_in "$sbatch --help \| $bin_grep test"
	set matches 0
	spawn $file_in
	expect {
	-re "Registered component of slurm test suite" {
	incr matches
	exp_continue
	}
	timeout {
	fail "srun not responding"
	}
	eof {
	wait
	}
	}
	if {$matches != 1} {
	fail "Spank help message not in sbatch help message ($matches != 1)"
	}

	#
	# Test of locally logged messages().
	# Add sleep for NFS delays in propagating $spank_conf_file
	#
	# NOTE: This test will fail if plugstack.conf is not in a shared location
	# The login node (where test is started) will modify the file and the
	# slurmd/slurmstepd on the compute node will not get the updated
	# configuration.
	#
	log_info "Test locally logged messages........."

	log_debug "Checksum of local $spank_conf_file"
	spawn $bin_sum $spank_conf_file
	expect {
	-re timeout {
	fail "sbatch not responding"
	}
	eof {
	wait
	}
	}

	sleep 30
	make_bash_script $file_in "
	$srun --test_suite_srun=5 $bin_sum $spank_conf_file
	"

	set matches 0
	spawn $sbatch --test_suite_sbatch=4 -N1 -t1 -o $file_out $file_in
	expect {
	-re "_test_opt_process_sbatch: opt_arg_sbatch=4" {
	incr matches
	exp_continue
	}
	-re "Submitted batch job ($number)" {
	set job_id $expect_out(1,string)
	exp_continue
	}
	-re "slurm_spank_exit: opt_arg_sbatch=4 opt_arg_srun=0" {
	incr matches
	exp_continue
	}
	-re timeout {
	fail "sbatch not responding"
	}
	eof {
	wait
	}
	}
	if {$job_id == 0} {
	fail "Batch submit failure"
	}
	if {$matches != 2} {
	fail "Spank options not processed by sbatch ($matches != 2)"
	}

	#
	# Wait for job to complete and check for files
	#
	wait_for_job -fail $job_id "DONE"

	# NOTE: spank logs from sbatch and srun would be intermingled here
	wait_for_file -fail $file_out
	set matches 0
	set matches_sbatch 0
	set matches_srun 0
	spawn $bin_cat $file_out
	expect {
	-re "error" {
	fail "Some error happened"
	}
	-re "_test_opt_process_(.*?=$number)" {
	set substr $expect_out(1,string)
	if {[string equal "sbatch: opt_arg_sbatch=4" $substr]} {
	incr matches_sbatch
	} elseif {[string equal "srun: opt_arg_srun=5" $substr]} {
	incr matches_srun
	}
	exp_continue
	}
	-re "slurm_spank_local_user_init" {
	incr matches
	exp_continue
	}
	-re "slurm_spank_exit: opt_arg_sbatch=4 opt_arg_srun=5" {
	incr matches
	exp_continue
	}
	eof {
	wait
	}
	}
	if {$matches_sbatch != 1} {
	fail "Local (srun) sbatch spank plugin failure ($matches_sbatch != 1)"
	} elseif {$matches_srun != 1} {
	fail "Local (srun) srun spank plugin failure ($matches_srun != 1)"
	} elseif {$matches != 2} {
	fail "Local (srun) spank plugin failure ($matches != 2)"
	} else {
	log_debug "Local (srun) spank plugin success"
	}

	#
	# Test for slurmd output in file
	#
	wait_for_file -fail $spank_out
	set matches 0
	set matches_sbatch 0
	set matches_srun 0
	set matches_spank_exit 0
	set matches_spank_init 0

	set expected_spank_exit 2
	set expected_spank_init 2
	set expected_sbatch 2
	set expected_srun 2
	set expected_items 5

	# If PrologFlags=contain is in slurm.conf, then an "extern" step will be
	# launched on each node, so we need to check for additional output
	log_user 0
	spawn $scontrol show config
	expect {
	-re "PrologFlags\\s=\\s\[A-z/,\]*Contain" {
	incr expected_spank_exit
	}
	timeout {
	fail "scontrol show config not responding"
	}
	eof {
	wait
	}
	}
	log_user 1

	spawn $bin_sort $spank_out
	expect {
	-re "slurm_spank_(\\S+): opt_arg_sbatch=($number) opt_arg_srun=($number)" {
	set spank_type $expect_out(1,string)
	set sbatch_arg $expect_out(2,string)
	set srun_arg $expect_out(3,string)

	if {$spank_type eq "exit"} {
	incr matches_spank_exit
	} elseif {$spank_type eq "task_init"} {
	incr matches_spank_init
	}

	if {($sbatch_arg eq "4") && ($srun_arg eq "0")} {
	incr matches_sbatch
	} elseif {($sbatch_arg eq "4") && ($srun_arg eq "5")} {
	incr matches_srun
	}
	exp_continue
	}
	-re "spank_get_item: argv" {
	incr matches
	exp_continue
	}
	-re "spank_get_item: my_uid=" {
	incr matches
	exp_continue
	}
	eof {
	wait
	}
	}

	if {$matches_spank_exit != $expected_spank_exit} {
	fail "Remote (slurmd) spank_exit spank plugin failure ($matches_spank_exit != $expected_spank_exit)"
	}
	if {$matches_spank_init != $expected_spank_init} {
	fail "Remote (slurmd) spank_init spank plugin failure ($matches_spank_init != $expected_spank_init)"
	}
	if {$matches_sbatch != $expected_sbatch} {
	fail "Remote (slurmd) sbatch spank plugin failure ($matches_sbatch != $expected_sbatch)"
	}
	if {$matches_srun != $expected_srun} {
	fail "Remote (slurmd) srun spank plugin failure ($matches_srun != $expected_srun)"
	}
	if {$matches != $expected_items} {
	fail "Remote (slurmd) spank plugin failure ($matches != $expected_items)"
	}
	log_debug "Remote (slurmd) spank plugin success"

	# Test if internal spank option parsing works (Bug 10454)
	set output [run_command_output -timeout 2 -xfail -subtest "$sbatch --test_suite_sbatch=NaN --wrap='/bin/true'"]
	subtest {[regexp "sbatch: error: Non-zero argument required" $output]} "sbatch should print error message from spank"

	set output [run_command_output -timeout 2 -xfail -subtest "$salloc --test_suite_sbatch=NaN"]
	subtest {[regexp "salloc: error: Non-zero argument required" $output]} "salloc should print error message from spank"

	set output [run_command_output -timeout 2 -xfail -subtest "$srun --test_suite_sbatch=NaN"]
	subtest {[regexp "srun: error: Non-zero argument required" $output]} "srun should print error message from spank"