testsuite/expect/test1.86 - SchedMD/slurm - Git at Google

 #!/usr/bin/env expect
 ############################################################################
 # Purpose: Test of Slurm functionality
 #          Confirm node selection from within a job step on existing allocation
 #          (--nodelist, --exclude, --nodes and --nprocs options).
 ############################################################################
 # Copyright (C) 2002-2007 The Regents of the University of California.
 # Copyright (C) 2008-2009 Lawrence Livermore National Security.
 # Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
 # Written by Morris Jette <jette1@llnl.gov>
 # CODE-OCEC-09-009. All rights reserved.
 #
 # This file is part of Slurm, a resource management program.
 # For details, see <https://slurm.schedmd.com/>.
 # Please also read the included file: DISCLAIMER.
 #
 # Slurm is free software; you can redistribute it and/or modify it under
 # the terms of the GNU General Public License as published by the Free
 # Software Foundation; either version 2 of the License, or (at your option)
 # any later version.
 #
 # Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 # details.
 #
 # You should have received a copy of the GNU General Public License along
 # with Slurm; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
 ############################################################################
 source ./globals

 # find out if we have enough nodes to test functionality
 set node_count [get_partition_param [default_partition] "TotalNodes"]
 if { $node_count < 2 } {
 	skip "Insufficient nodes in default partition ($node_count < 2)"
 }

 #
 # Expect seems to have trouble parsing with this test due to a bug in Expect.
 # It overwrites the stdout parsed with stdin to the commands
 # Change PATH to shorten the strings, which seems to help
 #
 set env(PATH) "${slurm_dir}/bin:$env(PATH)"

 #
 # Submit a 2 node job
 #
 set job_id 0
 set timeout $max_job_delay
 spawn $salloc -N2 -t1 $bin_bash
 expect {
 	-re "Granted job allocation ($number)" {
 		set job_id $expect_out(1,string)
 		reset_bash_prompt
 		exp_continue
 	}
 	-re "salloc: error" {
 		skip "Can't test srun task distribution"
 	}
 	-re "Unable to contact" {
 		fail "Slurm appears to be down"
 	}
 	"$test_prompt" {
 		log_debug "Job initiated"
 	}
 	timeout {
 		fail "salloc not responding"
 	}
 	eof {
 		wait
 		fail "salloc terminated"
 	}
 }

 #
 # Get node names
 #
 set host_0      ""
 set host_1      ""
 send "$srun -l $bin_printenv SLURMD_NODENAME\r"
 expect {
 	-re "($number): *($re_word_str)" {
 		set host_inx $expect_out(1,string)
 		if {$host_inx == 0} {
 			set host_0 $expect_out(2,string)
 		}
 		if {$host_inx == 1} {
 			set host_1 $expect_out(2,string)
 		}
 		exp_continue
 	}
 	-re "Unable to contact" {
 		fail "Slurm appears to be down"
 	}
 	"$test_prompt" {
 		log_debug "srun completed"
 	}
 	timeout {
 		fail "srun not responding"
 	}
 	eof {
 		fail "srun EOF"
 	}
 }

 #
 # Verify node count
 #
 subtest {$host_0 ne ""} "Should get hostname of task 0"
 subtest {$host_1 ne ""} "Should get hostname of task 1"

 #
 # Exclude specific node
 #
 set matches 0
 send "$srun -l -N1 -n1 --exclude=$host_0 $bin_printenv SLURMD_NODENAME\r"
 expect {
 	-re "0: *($re_word_str)" {
 		if {$expect_out(1,string) eq $host_1} {
 			incr matches
 		} else {
 			fail "Wrong node responded"
 		}
 		exp_continue
 	}
 	-re "Unable to contact" {
 		fail "Slurm appears to be down"
 	}
 	"$test_prompt" {
 		log_debug "srun completed"
 	}
 	timeout {
 		fail "srun not responding"
 	}
 	eof {
 		fail "srun EOF"
 	}
 }

 subtest {$matches != 0} "Required node should respond"

 #
 # Exclude specific node
 #
 set matches 0
 send "$srun -l -N1 -n1 --exclude=$host_1 $bin_printenv SLURMD_NODENAME\r"
 expect {
 	-re "0: *($re_word_str)" {
 		if {$expect_out(1,string) eq $host_0} {
 			incr matches
 		} else {
 			fail "Wrong node responded"
 		}
 		exp_continue
 	}
 	-re "Unable to contact" {
 		fail "Slurm appears to be down"
 	}
 	"$test_prompt" {
 		log_debug "srun completed"
 	}
 	timeout {
 		fail "srun not responding"
 	}
 	eof {
 		fail "srun EOF"
 	}
 }

 subtest {$matches != 0} "Required node should respond"

 #
 # Include specific node
 #
 set matches 0
 send "$srun -l -N1 -n1 --nodelist=$host_0 $bin_printenv SLURMD_NODENAME\r"
 expect {
 	-re "Requested node configuration is not available" {
 		send "exit\r"
 		skip "Appears you are using multiple slurmd testing. This test won't work in that fashion"
 	}
 	-re "0: *($re_word_str)" {
 		if {$expect_out(1,string) eq $host_0} {
 			incr matches
 		} else {
 			fail "Wrong node responded"
 		}
 		exp_continue
 	}
 	-re "Unable to contact" {
 		fail "Slurm appears to be down"
 	}
 	"$test_prompt" {
 		log_debug "srun completed"
 	}
 	timeout {
 		fail "srun not responding"
 	}
 	eof {
 		fail "srun EOF"
 	}
 }

 subtest {$matches != 0} "Required node should respond"

 #
 # Include specific node
 #
 set matches 0
 send "$srun -l -N1 -n1 --nodelist=$host_1 $bin_printenv SLURMD_NODENAME\r"
 expect {
 	-re "0: *($re_word_str)" {
 		if {$expect_out(1,string) eq $host_1} {
 			incr matches
 		} else {
 			fail "Wrong node responded"
 		}
 		exp_continue
 	}
 	-re "Unable to contact" {
 		fail "Slurm appears to be down"
 	}
 	"$test_prompt" {
 		log_debug "srun completed"
 	}
 	timeout {
 		fail "srun not responding"
 	}
 	eof {
 		fail "srun EOF"
 	}
 }

 subtest {$matches != 0} "Required node should respond"

 #
 # Error test: Overlapping include/exclude node list
 # NOTE: some versions of Expect have a problem with long input
 #	lines, so we use two send commands for this execute line
 #
 set matches 0
 send "$srun -l -N1 -n1 --nodelist=$host_0 "
 send " --exclude=$host_0 $bin_printenv SLURMD_NODENAME\r"
 expect {
 	-re "0: *($re_word_str)" {
 		fail "Wrong node responded"
 	}
 	"error:" {
 		log_debug "This error is expected, no worries"
 		incr matches
 		exp_continue
 	}
 	"$test_prompt" {
 		log_debug "srun completed"
 	}
 	timeout {
 		fail "srun not responding"
 	}
 	eof {
 		fail "srun EOF"
 	}
 }

 subtest {$matches != 0} "Expected error for overlapping include/exclude node list"

 #
 # Error test: Exceed node count
 #
 set matches 0
 send "$srun -l -N3 -n3 -O $bin_printenv SLURMD_NODENAME\r"
 expect {
 	-re "0: *($re_word_str)" {
 		fail "Wrong node responded"
 	}
 	"error:" {
 		log_debug "This error is expected, no worries"
 		incr matches
 		exp_continue
 	}
 	"$test_prompt" {
 		log_debug "srun completed"
 	}
 	timeout {
 		fail "srun not responding"
 	}
 	eof {
 		fail "srun EOF"
 	}
 }

 subtest {$matches != 0} "Expected error for exceeding node count"

 #
 # Run with fewer nodes
 #
 set test_0      ""
 set test_1      ""
 send "$srun -l -N1-1 -n2 -O $bin_printenv SLURMD_NODENAME\r"
 expect {
 	-re "($number)\\.($number):($re_word_str)" {
 		set test_0 $expect_out(3,string)
 		set test_1 $expect_out(3,string)
 		exp_continue
 	}
 	-re "($number): ($re_word_str)" {
 		set host_inx $expect_out(1,string)
 		if {$host_inx == 0} {
 			set test_0 $expect_out(2,string)
 		}
 		if {$host_inx == 1} {
 			set test_1 $expect_out(2,string)
 		}
 		exp_continue
 	}
 	-re "Unable to contact" {
 		fail "Slurm appears to be down"
 	}
 	"$test_prompt" {
 		log_debug "srun completed"
 	}
 	timeout {
 		fail "srun not responding"
 	}
 	eof {
 		fail "srun EOF"
 	}
 }

 subtest {$test_0 eq $test_1} "Expecting only a single node to respond"

 #
 # Error test: Exceed task count, first get the processor count then exceed it
 #
 set processors 1
 send "$srun -l -c1 $bin_printenv SLURMD_NODENAME\r"
 expect {
 	-re "($number)-($number):($re_word_str)" {
 		set processors [expr $processors + $expect_out(2,string) - $expect_out(1,string) + 1]
 		exp_continue
 	}
 	-re "($number): *($re_word_str)" {
 		incr processors
 		exp_continue
 	}
 	-re "Unable to contact" {
 		fail "Slurm appears to be down"
 	}
 	"$test_prompt" {
 		log_debug "srun completed"
 	}
 	timeout {
 		fail "srun not responding"
 	}
 	eof {
 		fail "srun EOF"
 	}
 }
 set matches 0
 send "$srun -l -n $processors $bin_printenv SLURMD_NODENAME\r"
 expect {
 	-re "error:" {
 		incr matches
 		exp_continue
 	}
 	-re "$job_id:" {
 #		Avoid confusiuon with task ID below
 		exp_continue
 	}
 	-re "($number): *($re_word_str)" {
 		fail "wrong node responded"
 	}
 	"$test_prompt" {
 		log_debug "srun completed"
 	}
 	timeout {
 		fail "srun not responding"
 	}
 	eof {
 		fail "srun EOF"
 	}
 }

 subtest {$matches != 0} "Expecting error for exceeding processor count"

 #
 # Post-processing
 #
 send "exit\r"
 expect {
 	-re "error.*Exit 1" {
 		log_info "This error is expected, no worries"
 		exp_continue
 	}
 	timeout {
 		fail "srun not responding"
 	}
 	eof {
 		wait
 	}
 }
	#!/usr/bin/env expect
	############################################################################
	# Purpose: Test of Slurm functionality
	# Confirm node selection from within a job step on existing allocation
	# (--nodelist, --exclude, --nodes and --nprocs options).
	############################################################################
	# Copyright (C) 2002-2007 The Regents of the University of California.
	# Copyright (C) 2008-2009 Lawrence Livermore National Security.
	# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
	# Written by Morris Jette <jette1@llnl.gov>
	# CODE-OCEC-09-009. All rights reserved.
	#
	# This file is part of Slurm, a resource management program.
	# For details, see <https://slurm.schedmd.com/>.
	# Please also read the included file: DISCLAIMER.
	#
	# Slurm is free software; you can redistribute it and/or modify it under
	# the terms of the GNU General Public License as published by the Free
	# Software Foundation; either version 2 of the License, or (at your option)
	# any later version.
	#
	# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
	# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
	# details.
	#
	# You should have received a copy of the GNU General Public License along
	# with Slurm; if not, write to the Free Software Foundation, Inc.,
	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	############################################################################
	source ./globals

	# find out if we have enough nodes to test functionality
	set node_count [get_partition_param [default_partition] "TotalNodes"]
	if { $node_count < 2 } {
	skip "Insufficient nodes in default partition ($node_count < 2)"
	}

	#
	# Expect seems to have trouble parsing with this test due to a bug in Expect.
	# It overwrites the stdout parsed with stdin to the commands
	# Change PATH to shorten the strings, which seems to help
	#
	set env(PATH) "${slurm_dir}/bin:$env(PATH)"

	#
	# Submit a 2 node job
	#
	set job_id 0
	set timeout $max_job_delay
	spawn $salloc -N2 -t1 $bin_bash
	expect {
	-re "Granted job allocation ($number)" {
	set job_id $expect_out(1,string)
	reset_bash_prompt
	exp_continue
	}
	-re "salloc: error" {
	skip "Can't test srun task distribution"
	}
	-re "Unable to contact" {
	fail "Slurm appears to be down"
	}
	"$test_prompt" {
	log_debug "Job initiated"
	}
	timeout {
	fail "salloc not responding"
	}
	eof {
	wait
	fail "salloc terminated"
	}
	}

	#
	# Get node names
	#
	set host_0 ""
	set host_1 ""
	send "$srun -l $bin_printenv SLURMD_NODENAME\r"
	expect {
	-re "($number): *($re_word_str)" {
	set host_inx $expect_out(1,string)
	if {$host_inx == 0} {
	set host_0 $expect_out(2,string)
	}
	if {$host_inx == 1} {
	set host_1 $expect_out(2,string)
	}
	exp_continue
	}
	-re "Unable to contact" {
	fail "Slurm appears to be down"
	}
	"$test_prompt" {
	log_debug "srun completed"
	}
	timeout {
	fail "srun not responding"
	}
	eof {
	fail "srun EOF"
	}
	}

	#
	# Verify node count
	#
	subtest {$host_0 ne ""} "Should get hostname of task 0"
	subtest {$host_1 ne ""} "Should get hostname of task 1"

	#
	# Exclude specific node
	#
	set matches 0
	send "$srun -l -N1 -n1 --exclude=$host_0 $bin_printenv SLURMD_NODENAME\r"
	expect {
	-re "0: *($re_word_str)" {
	if {$expect_out(1,string) eq $host_1} {
	incr matches
	} else {
	fail "Wrong node responded"
	}
	exp_continue
	}
	-re "Unable to contact" {
	fail "Slurm appears to be down"
	}
	"$test_prompt" {
	log_debug "srun completed"
	}
	timeout {
	fail "srun not responding"
	}
	eof {
	fail "srun EOF"
	}
	}

	subtest {$matches != 0} "Required node should respond"

	#
	# Exclude specific node
	#
	set matches 0
	send "$srun -l -N1 -n1 --exclude=$host_1 $bin_printenv SLURMD_NODENAME\r"
	expect {
	-re "0: *($re_word_str)" {
	if {$expect_out(1,string) eq $host_0} {
	incr matches
	} else {
	fail "Wrong node responded"
	}
	exp_continue
	}
	-re "Unable to contact" {
	fail "Slurm appears to be down"
	}
	"$test_prompt" {
	log_debug "srun completed"
	}
	timeout {
	fail "srun not responding"
	}
	eof {
	fail "srun EOF"
	}
	}

	subtest {$matches != 0} "Required node should respond"

	#
	# Include specific node
	#
	set matches 0
	send "$srun -l -N1 -n1 --nodelist=$host_0 $bin_printenv SLURMD_NODENAME\r"
	expect {
	-re "Requested node configuration is not available" {
	send "exit\r"
	skip "Appears you are using multiple slurmd testing. This test won't work in that fashion"
	}
	-re "0: *($re_word_str)" {
	if {$expect_out(1,string) eq $host_0} {
	incr matches
	} else {
	fail "Wrong node responded"
	}
	exp_continue
	}
	-re "Unable to contact" {
	fail "Slurm appears to be down"
	}
	"$test_prompt" {
	log_debug "srun completed"
	}
	timeout {
	fail "srun not responding"
	}
	eof {
	fail "srun EOF"
	}
	}

	subtest {$matches != 0} "Required node should respond"

	#
	# Include specific node
	#
	set matches 0
	send "$srun -l -N1 -n1 --nodelist=$host_1 $bin_printenv SLURMD_NODENAME\r"
	expect {
	-re "0: *($re_word_str)" {
	if {$expect_out(1,string) eq $host_1} {
	incr matches
	} else {
	fail "Wrong node responded"
	}
	exp_continue
	}
	-re "Unable to contact" {
	fail "Slurm appears to be down"
	}
	"$test_prompt" {
	log_debug "srun completed"
	}
	timeout {
	fail "srun not responding"
	}
	eof {
	fail "srun EOF"
	}
	}

	subtest {$matches != 0} "Required node should respond"

	#
	# Error test: Overlapping include/exclude node list
	# NOTE: some versions of Expect have a problem with long input
	# lines, so we use two send commands for this execute line
	#
	set matches 0
	send "$srun -l -N1 -n1 --nodelist=$host_0 "
	send " --exclude=$host_0 $bin_printenv SLURMD_NODENAME\r"
	expect {
	-re "0: *($re_word_str)" {
	fail "Wrong node responded"
	}
	"error:" {
	log_debug "This error is expected, no worries"
	incr matches
	exp_continue
	}
	"$test_prompt" {
	log_debug "srun completed"
	}
	timeout {
	fail "srun not responding"
	}
	eof {
	fail "srun EOF"
	}
	}

	subtest {$matches != 0} "Expected error for overlapping include/exclude node list"

	#
	# Error test: Exceed node count
	#
	set matches 0
	send "$srun -l -N3 -n3 -O $bin_printenv SLURMD_NODENAME\r"
	expect {
	-re "0: *($re_word_str)" {
	fail "Wrong node responded"
	}
	"error:" {
	log_debug "This error is expected, no worries"
	incr matches
	exp_continue
	}
	"$test_prompt" {
	log_debug "srun completed"
	}
	timeout {
	fail "srun not responding"
	}
	eof {
	fail "srun EOF"
	}
	}

	subtest {$matches != 0} "Expected error for exceeding node count"

	#
	# Run with fewer nodes
	#
	set test_0 ""
	set test_1 ""
	send "$srun -l -N1-1 -n2 -O $bin_printenv SLURMD_NODENAME\r"
	expect {
	-re "($number)\\.($number):($re_word_str)" {
	set test_0 $expect_out(3,string)
	set test_1 $expect_out(3,string)
	exp_continue
	}
	-re "($number): ($re_word_str)" {
	set host_inx $expect_out(1,string)
	if {$host_inx == 0} {
	set test_0 $expect_out(2,string)
	}
	if {$host_inx == 1} {
	set test_1 $expect_out(2,string)
	}
	exp_continue
	}
	-re "Unable to contact" {
	fail "Slurm appears to be down"
	}
	"$test_prompt" {
	log_debug "srun completed"
	}
	timeout {
	fail "srun not responding"
	}
	eof {
	fail "srun EOF"
	}
	}

	subtest {$test_0 eq $test_1} "Expecting only a single node to respond"

	#
	# Error test: Exceed task count, first get the processor count then exceed it
	#
	set processors 1
	send "$srun -l -c1 $bin_printenv SLURMD_NODENAME\r"
	expect {
	-re "($number)-($number):($re_word_str)" {
	set processors [expr $processors + $expect_out(2,string) - $expect_out(1,string) + 1]
	exp_continue
	}
	-re "($number): *($re_word_str)" {
	incr processors
	exp_continue
	}
	-re "Unable to contact" {
	fail "Slurm appears to be down"
	}
	"$test_prompt" {
	log_debug "srun completed"
	}
	timeout {
	fail "srun not responding"
	}
	eof {
	fail "srun EOF"
	}
	}
	set matches 0
	send "$srun -l -n $processors $bin_printenv SLURMD_NODENAME\r"
	expect {
	-re "error:" {
	incr matches
	exp_continue
	}
	-re "$job_id:" {
	# Avoid confusiuon with task ID below
	exp_continue
	}
	-re "($number): *($re_word_str)" {
	fail "wrong node responded"
	}
	"$test_prompt" {
	log_debug "srun completed"
	}
	timeout {
	fail "srun not responding"
	}
	eof {
	fail "srun EOF"
	}
	}

	subtest {$matches != 0} "Expecting error for exceeding processor count"

	#
	# Post-processing
	#
	send "exit\r"
	expect {
	-re "error.*Exit 1" {
	log_info "This error is expected, no worries"
	exp_continue
	}
	timeout {
	fail "srun not responding"
	}
	eof {
	wait
	}
	}