testsuite/expect/test1.19

#!/usr/bin/expect
############################################################################
# Purpose: Test of SLURM functionality
#          Test srun stdout/err file name formatting (--output and --error 
#          options with %j, %J, %n, %s and %t specifications).
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
#
# Note:    This script generates and then deletes files in the working directory
#          named test1.19.*.output and test1.19.*.error
############################################################################
# Copyright (C) 2002-2006 The Regents of the University of California.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Morris Jette <jette1@llnl.gov>
# UCRL-CODE-217948.
# 
# This file is part of SLURM, a resource management program.
# For details, see <http://www.llnl.gov/linux/slurm/>.
#  
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
# 
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
# 
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_id          "1.19"
set exit_code        0
set file_err_j       "test$test_id.j.%j.error"
set file_in          "test$test_id.input"
set file_out_J       "test$test_id.J.%J.output"
set file_out_n       "test$test_id.n.%n.output"
set file_out_s       "test$test_id.s.%s.output"
set file_out_t       "test$test_id.t.%t.output"
set job_id           0
set task_cnt         5

print_header $test_id

#
# Spawn a shell via srun that includes "task_id" (%t) in stdout file name
# and confirm they are created
#
for {set task_id 0} {$task_id < $task_cnt} {incr task_id} {
	set file_out_t_glob  "test$test_id.t.$task_id.output"
	exec $bin_rm -f $file_out_t_glob
}
set timeout $max_job_delay
spawn $srun --output=$file_out_t -N1 -n$task_cnt -O -v -t1 $bin_id
expect {
	-re "jobid ($number).*" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: srun not responding\n"
		kill_srun
		exit 1
	}
	eof {
		wait
	}
}
if {$job_id == 0} {
	send_user "\nFAILURE: job initiation failed\n"
	exit 1
}
set file_cnt 0
for {set task_id 0} {$task_id < $task_cnt} {incr task_id} {
	set file_out_t_glob  "test$test_id.t.$task_id.output"
	if {[wait_for_file $file_out_t_glob] != 0} {
		set exit_code 1
	}
	spawn $bin_rm -i $file_out_t_glob
	expect {
		-re "\[Rr\]emove .*?" {
			send "y\n"
			incr file_cnt
			exp_continue
		}
		eof {
			wait
		}
	}
}
if {$file_cnt != $task_cnt} {
	send_user "\nFAILURE: file format of %t in stdout failed\n"
	set exit_code 1
}

#
# Spawn a shell via srun that includes "jobid" (%j) in stderr 
# file name and confirm it is created
#
set job_id   0
set srun_exit 0
spawn $srun --error=$file_err_j --output=/dev/null -N1 -n$task_cnt -O -v -t1 $bin_sleep aaa 
expect {
	-re "jobid ($number).*" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	-re "exit code $number" {
		send_user "This error is expected, no worries\n"
		set srun_exit 1
		exp_continue;
	}
	timeout {
		send_user "\nFAILURE: srun not responding\n"
		kill_srun
		exit 1
	}
	eof {
		wait
	}
}
if {$job_id == 0} {
	send_user "\nFAILURE: job initiation failed\n"
	exit 1
}
if {$srun_exit == 0} {
	send_user "\nFAILURE: exit code failed to indicate an error\n"
	set exit_code 1
}

set file_err_j_glob  "test$test_id.j.$job_id.error"
set file_cnt    0
spawn $bin_rm -i $file_err_j_glob
expect {
	-re "\[Rr\]emove .*?" {
		send "y\n"
		incr file_cnt
		exp_continue
	}
	eof {
		wait
	}
}
if {$file_cnt != 1} {
	send_user "\nFAILURE: file format of %j in stderr failed\n"
	set exit_code 1
}

#
# Spawn a shell via srun that includes "job_id.step_id" (%J) in stdout
#  file name and confirm it is created
#
set job_id   0
spawn $srun --output=$file_out_J -N1 -v -t1 $bin_hostname
expect {
	-re "jobid ($number).*" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: srun not responding\n"
		kill_srun
		exit 1
	}
	eof {
		wait
	}
}
if {$job_id == 0} {
	send_user "\nFAILURE: job initiation failed\n"
	exit 1
}
set file_cnt 0
set task_id  0
set file_out_J_glob  "test$test_id.J.$job_id.$task_id.output"
if {[wait_for_file $file_out_J_glob] != 0} {
	set exit_code 1
}
spawn $bin_rm -i $file_out_J_glob
expect {
	-re "\[Rr\]emove .*?" {
		send "y\n"
		incr file_cnt
		exp_continue
	}
	eof {
		wait
	}
}
if {$file_cnt != 1} {
	send_user "\nFAILURE: file format of %J in stdout failed\n"
	set exit_code 1
}

#
# Spawn a shell via srun that includes "node_id" (%n) in stdout
#  file name and confirm it is created
#
set node_id 0
set file_out_n_glob  "test$test_id.n.$node_id.output"
exec $bin_rm -f $file_out_n_glob

set job_id   0
spawn $srun --output=$file_out_n -N1 -n2 -O -v -t1 $bin_hostname
expect {
	-re "jobid ($number).*" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: srun not responding\n"
		kill_srun
		exit 1
	}
	eof {
		wait
	}
}
if {$job_id == 0} {
	send_user "\nFAILURE: job initiation failed\n"
	exit 1
}
set file_cnt 0
for {set node_id 0} {$node_id < 2} {incr node_id} {
	set file_out_n_glob  "test$test_id.n.$node_id.output"

	if {($node_id == 0) && ([wait_for_file $file_out_n_glob] != 0)} {
		send_user "\nFAILURE: Missing file $file_out_n_glob\n"
		set exit_code 1
	} else {
		exec $bin_sleep 1
	}
	if [file exists $file_out_n_glob] {
		spawn $bin_rm -i $file_out_n_glob
		expect {
			-re "\[Rr\]emove .*?" {
				send "y\n"
				incr file_cnt
				exp_continue
			}
			eof {
				wait
			}
		}
	}
}

if {$file_cnt != 1} {
	send_user "\nFAILURE: file format of %n in stdout failed\n"
	set exit_code 1
}

#
# Spawn a shell via srun that includes "step_id" (%s) in stdout
#  file name and confirm it is created. Use two step batch job.
#
exec $bin_rm -f $file_in
make_bash_script $file_in "
  $srun -n4 -O --output=$file_out_s $bin_hostname
  $srun -n4 -O --output=$file_out_s $bin_hostname
"

for {set step_id 0} {$step_id < 4} {incr step_id} {
	set file_out_s_glob  "test$test_id.s.$step_id.output"
	exec $bin_rm -f $file_out_s_glob
}

if { [test_bluegene] } {
	set node_cnt 32-2048
	set task_cnt 32
} else {
	if { [test_xcpu] } {
		set node_cnt 1-1
	} else {
		set node_cnt 1-4
	}
	set task_cnt 4
}

set job_id   0
spawn $srun --batch --output=/dev/null -N$node_cnt -n$task_cnt -O -t1 $file_in
expect {
	-re "jobid ($number) submitted" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: srun not responding\n"
		kill_srun
		exit 1
	}
	eof {
		wait
	}
}
if {$job_id == 0} {
	send_user "\nFAILURE: job initiation failed\n"
	exit 1
} else {
	if {[wait_for_job $job_id DONE] != 0} {
		send_user "\nFAILURE: error on job\n"
		set exit_code 1
	}
}
exec $bin_rm -f $file_in

set file_cnt 0
for {set step_id 0} {$step_id < 3} {incr step_id} {
	set file_out_s_glob  "test$test_id.s.$step_id.output"

	if {($step_id < 2) && ([wait_for_file $file_out_s_glob] != 0)} {
		send_user "\nFAILURE: Missing file $file_out_s_glob\n"
		set exit_code 1
	} else {
		exec $bin_sleep 1
	}
	if [file exists $file_out_s_glob] {
		spawn $bin_rm -i $file_out_s_glob
		expect {
			-re "\[Rr\]emove .*?" {
				send "y\n"
				incr file_cnt
				exp_continue
			}
			eof {
				wait
			}
		}
	}
}

if {$file_cnt != 2} {
	send_user "\nFAILURE: file format of %s in stdout failed($file_cnt)\n"
	set exit_code 1
}

#
# Post-processing
#
if {$exit_code == 0} {
	send_user "\nSUCCESS\n"
}
exit $exit_code