testsuite/expect/test1.48

#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
#          Test of mail options (--mail-type and --mail-user options).
############################################################################
# Copyright (C) 2002 The Regents of the University of California.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Morris Jette <jette1@llnl.gov>
# CODE-OCEC-09-009. All rights reserved.
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set got_job_grps         0
set got_login_grps       0
set config_dir           [get_conf_path]
set config_file          "$config_dir/slurm.conf"
set cwd                  "[$bin_pwd]"
set mail_prog            "$test_id.MailProg.sh"
set mail_prog_out        "$test_id.MailProg.out"

if {![is_super_user]} {
	skip "Can not test more unless SlurmUser or root"
}

proc cleanup { } {
	global config_file bin_rm mail_prog mail_prog_out job_id

	cancel_job $job_id
	restore_conf $config_file
	exec $bin_rm -f $mail_prog $mail_prog_out
	reconfigure
}

save_conf $config_file
make_bash_script $mail_prog "
echo $@ SLURM_JOB_ID=\$SLURM_JOB_ID SLURM_JOB_USER=\$SLURM_JOB_USER SLURM_JOB_MAIL_TYPE=\$SLURM_JOB_MAIL_TYPE SLURM_JOB_QUEUED_TIME=\$SLURM_JOB_QUEUED_TIME SLURM_JOB_RUN_TIME=\$SLURM_JOB_RUN_TIME SLURM_JOB_EXIT_CODE_MAX=\$SLURM_JOB_EXIT_CODE_MAX >> $cwd/$mail_prog_out
"

#Comment MailProg line if existed before
exec $bin_sed -i /^\[\t\s\]*MailProg\[\t\s\]*=/Id $config_file

# Append MailProg config to the slurm.conf
exec $bin_echo "MailProg=$cwd/$mail_prog" >> $config_file
reconfigure


#
# Execute 'id' to determine my user and group ID...
#
set login_grp_info [get_my_user_name]

#
# Submit a slurm job that will execute 'id'
#
set job_id 0
set got_job_id 0
set timeout $max_job_delay
spawn $srun -N1 -t1 --job-name=$test_name --mail-type=all --mail-user=$login_grp_info /bin/bash -c "echo \$SLURM_JOB_ID"
expect {
	-re "error" {
		fail "Error running srun"
	}
	-re "($number)" {
		set job_id $expect_out(1,string)
		set got_job_id 1
		exp_continue
	}
	timeout {
		fail "srun not responding"
	}
	eof {
		wait
	}
}

if {$got_job_id == 0} {
	fail "Did not get JobId info from slurm job"
}

# Wait for mails to go through agent
wait_for_file -fail $mail_prog_out

subtest {![wait_for_command_match "$bin_cat $mail_prog_out" "-s Slurm Job_id=$job_id Name=$test_name Began, \\S+ \\S+ \\S+ $login_grp_info SLURM_JOB_ID=$job_id SLURM_JOB_USER=$login_grp_info SLURM_JOB_MAIL_TYPE=Began SLURM_JOB_QUEUED_TIME=\\S+ SLURM_JOB_RUN_TIME= SLURM_JOB_EXIT_CODE_MAX="]} "Began mail output should be found in log"

subtest {![wait_for_command_match "$bin_cat $mail_prog_out" "-s Slurm Job_id=$job_id Name=$test_name Ended, \\S+ \\S+ \\S+, \\S+, \\S+ \\d+ $login_grp_info SLURM_JOB_ID=$job_id SLURM_JOB_USER=$login_grp_info SLURM_JOB_MAIL_TYPE=Ended SLURM_JOB_QUEUED_TIME= SLURM_JOB_RUN_TIME=\\S+ SLURM_JOB_EXIT_CODE_MAX=0"]} "Ended mail output should be found in log"

exec $bin_rm -f $mail_prog_out

# Wait for mails to go through agent
set job_id [submit_job "-N1 -t1 --job-name=$test_name.CANCELLED --mail-type=END --mail-user=$login_grp_info --wrap='sleep 100'"]
cancel_job $job_id
if {[wait_for_file $mail_prog_out] != 0} {
	fail "No output file"
}

subtest {![wait_for_command_match "$bin_cat $mail_prog_out" "-s Slurm Job_id=$job_id Name=$test_name.CANCELLED Ended, Run time \\S+, CANCELLED, ExitCode 0 $login_grp_info SLURM_JOB_ID=$job_id SLURM_JOB_USER=$login_grp_info SLURM_JOB_MAIL_TYPE=Ended SLURM_JOB_QUEUED_TIME= SLURM_JOB_RUN_TIME=\\S+ SLURM_JOB_EXIT_CODE_MAX=0"]} "Cancelled job mail output should be found in log"

exec $bin_rm -f $mail_prog_out
log_info "This will take ~60 seconds, since we're waiting for job timeout"
set job_id [submit_job -none -timeout 120 "-W -N1 -t1 --job-name=$test_name.TIMEDOUT --mail-type=ALL,TIME_LIMIT,TIME_LIMIT_90,TIME_LIMIT_80,TIME_LIMIT_50 --mail-user=$login_grp_info --wrap='sleep 300'"]
if {[wait_for_file $mail_prog_out] != 0} {
	fail "No output file"
}

subtest {![wait_for_command_match "$bin_cat $mail_prog_out" "-s Slurm Job_id=$job_id Name=$test_name.TIMEDOUT Began, Queued time \\S+ $login_grp_info SLURM_JOB_ID=$job_id SLURM_JOB_USER=$login_grp_info SLURM_JOB_MAIL_TYPE=Began SLURM_JOB_QUEUED_TIME=\\S+ SLURM_JOB_RUN_TIME= SLURM_JOB_EXIT_CODE_MAX="]} "Began mail output should be found in the log of timedout out job"
subtest {![wait_for_command_match "$bin_cat $mail_prog_out" "-s Slurm Job_id=$job_id Name=$test_name.TIMEDOUT Reached 50% of time limit, Run time \\S+ $login_grp_info SLURM_JOB_ID=$job_id SLURM_JOB_USER=$login_grp_info SLURM_JOB_MAIL_TYPE=Reached 50% of time limit SLURM_JOB_QUEUED_TIME= SLURM_JOB_RUN_TIME=\\S+ SLURM_JOB_EXIT_CODE_MAX="]} "TIME_LIMIT_50 mail output should be found in log for timed out job"
subtest {![wait_for_command_match "$bin_cat $mail_prog_out" "-s Slurm Job_id=$job_id Name=$test_name.TIMEDOUT Reached 80% of time limit, Run time \\S+ $login_grp_info SLURM_JOB_ID=$job_id SLURM_JOB_USER=$login_grp_info SLURM_JOB_MAIL_TYPE=Reached 80% of time limit SLURM_JOB_QUEUED_TIME= SLURM_JOB_RUN_TIME=\\S+ SLURM_JOB_EXIT_CODE_MAX="]} "TIME_LIMIT_80 mail output should be found in log for timed out job"
subtest {![wait_for_command_match "$bin_cat $mail_prog_out" "-s Slurm Job_id=$job_id Name=$test_name.TIMEDOUT Reached 90% of time limit, Run time \\S+ $login_grp_info SLURM_JOB_ID=$job_id SLURM_JOB_USER=$login_grp_info SLURM_JOB_MAIL_TYPE=Reached 90% of time limit SLURM_JOB_QUEUED_TIME= SLURM_JOB_RUN_TIME=\\S+ SLURM_JOB_EXIT_CODE_MAX="]} "TIME_LIMIT_90 mail output should be found in log for timed out job"
subtest {![wait_for_command_match "$bin_cat $mail_prog_out" "-s Slurm Job_id=$job_id Name=$test_name.TIMEDOUT Reached time limit, Run time \\S+ $login_grp_info SLURM_JOB_ID=$job_id SLURM_JOB_USER=$login_grp_info SLURM_JOB_MAIL_TYPE=Reached time limit SLURM_JOB_QUEUED_TIME= SLURM_JOB_RUN_TIME=\\S+ SLURM_JOB_EXIT_CODE_MAX="]} "Time limit mail output should be found in log for timedout job"
subtest {![wait_for_command_match "$bin_cat $mail_prog_out" "-s Slurm Job_id=$job_id Name=$test_name.TIMEDOUT Failed, Run time \\S+, TIMEOUT, ExitCode 0 $login_grp_info SLURM_JOB_ID=$job_id SLURM_JOB_USER=$login_grp_info SLURM_JOB_MAIL_TYPE=Failed SLURM_JOB_QUEUED_TIME= SLURM_JOB_RUN_TIME=\\S+ SLURM_JOB_EXIT_CODE_MAX=0"]} "Failed mail output should be found in log for timedout job"

exec $bin_rm -f $mail_prog_out
log_info "Requeue job"
set job_id [ submit_job "-N1 -t1 --job-name=$test_name.REQUEUE --mail-type=REQUEUE --requeue --mail-user=$login_grp_info --wrap='sleep 300'"]
subtest {![wait_for_job $job_id RUNNING]} "$job_id should be running"
run_command -fail "$scontrol requeue $job_id"
if {[wait_for_file $mail_prog_out] != 0} {
	fail "No output file"
}
cancel_job $job_id

subtest {![wait_for_command_match "$bin_cat $mail_prog_out" "-s Slurm Job_id=$job_id Name=$test_name.REQUEUE Requeued, Run time \\S+ $login_grp_info SLURM_JOB_ID=$job_id SLURM_JOB_USER=$login_grp_info SLURM_JOB_MAIL_TYPE=Requeued"]} "Requeue mail output should be found in the log of requeued job"