blob: 4e0f1f4b3e0c05a65e0644a7858df531c2aa9bff [file] [log] [blame]
#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm sacct with -s R, -S and -E options
############################################################################
# Copyright (C) SchedMD LLC.
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
############################################################################
source ./globals
set timeout 60
set jobid 0
set jobid_dependent 0
set jobid_queue 0
############################################################################
#
# Proc: sacct_query
#
# Purpose: Runs sacct with the given args and subtest if job and expected_err
# is returned or not as expected.
#
# Input: the jobname to match, the args to pass to sacct, and booleans
# to specify if job is expected to be returned or not, and if an
# error is expected or not.
#
############################################################################
proc sacct_query {jobname sacct_args expected_job {expected_err false}} {
global sacct
set result [run_command "$sacct -v -X -p --format=job,jobname,eligible,start,end,state $sacct_args"]
set output [dict get $result output]
set job_found [regexp "($jobname)" $output]
set err_found [regexp "sacct: error:" $output]
regsub -- {-j [^ ]+} $sacct_args "-j <job_id>" filtered_args
regsub -- {-S [^ ]+} $filtered_args "-S <start_time>" filtered_args
regsub -- {-E [^ ]+} $filtered_args "-E <end_time>" filtered_args
if {$expected_job} {
subtest {$job_found} "Verify job is shown by sacct $filtered_args"
} else {
subtest {!$job_found} "Verify job is NOT shown by sacct $filtered_args"
}
if {$expected_err} {
subtest {$err_found} "Verify error message is printed by sacct $filtered_args"
subtest {[dict get $result exit_code]} "Verify exit code is non-zero for sacct $filtered_args"
} else {
subtest {!$err_found} "Verify error message is NOT printed by sacct $filtered_args"
subtest {![dict get $result exit_code]} "Verify exit code is zero for sacct $filtered_args"
}
}
############################################################################
#
# Proc: get_time
#
# Purpose: Obtain the current time with the right format
# (YYYY-MM-DDTHH:MM:SS) with the date command.
#
# Returns: The date, or 0.
#
############################################################################
proc get_time { } {
global bin_date
set output [run_command_output -fail "$bin_date '+%FT%T#'"]
if {![regexp "(.+)#" $output - date]} {
fail "Unable to parse date"
}
return $date
}
############################################################################
#
# Check preconditions
#
############################################################################
if {[get_config_param "AccountingStorageType"] ne "accounting_storage/slurmdbd"} {
skip "This test can't be run without a usable AccountStorageType."
}
proc cleanup {} {
global jobid jobid_dependent jobid_queue
cancel_job [list $jobid $jobid_dependent $jobid_queue]
}
############################################################################
#
# Main Test
#
# The following diagram shows the timeline of a running job used in this
# test:
#
# <-------S------>|<------E------>|<------P------>|<------R------>|<------C-------->
# ----|-----|----|----|-----|----|----|-----|----|----|-----|----|----|-----|----> t
# QsS QeS | QsE QeE | QsP QeP | QsR QeR | QsC QeC
# Job Submitted Job Eligible Job Starts Job Ends
#
# As you can see we define 3 time zones:
# S) Before job is Submitted
# E) Job is NonEligible
# P) Job is Pending (Eligible)
# R) Job is Running
# C) Job is Completed/Cancelled
#
# In each time zone we define two times to be used in the sacct queries:
#
# Qs[S,E,P,R,C]: Used mainly with -S
# Qe[S,E,P,R,C]: Used mainly with -E
#
# The test basically does:
#
# 1) Get the QsS and QeS
# 2) Submit a job forcing a Dependency (NonEligible) and later Pending
# 3) Get the QsE and QeE
# 4) Do all the test with Q[s,e][S,E] while the job is still NonEligible
# 3) Get the QsP and QeP
# 4) Do all the test with Q[s,e][S,E,P] while the job is still Pending (Eligible)
# 5) Make the job to Run
# 6) Get the QsR and QeR
# 7) Do all the test with Q[s,e][S,E,P,R] while the job is still Running
# 8) Cancel the job
# 9) Get the QsC and QeC
# 10) Do all the test with Q[s,e][S,E,P,R,C] with a Cancelled/Completed job.
#
# The expected behavior of default values of -S and -E are the ones defined
# in the function slurmdb_job_cond_def_start_end():
#
# - with -j and -s:
# -S defaults to Epoch 0
# -E defaults to -S (unless no -S then Now)
# - with only -j (NOT -s)
# -S defaults to Epoch 0
# -E defaults to Now
# - with only -s (NOT -j):
# -S defaults to Now
# -E defaults to -S
# - without either -j nor -s:
# -S defaults to Midnight
# -E defaults to Now
#
############################################################################
############################################################################
#
# Get Random name for the job, to avoid errors with old/duplicated jobs
#
set current_timestamp [clock seconds]
set jobname "test${test_id}-${current_timestamp}"
log_info "Unique job name used to avoid false test errors with old/duplicated jobs: $jobname."
#
# Get query times before any job is run (QsS and QeS)
#
set QsS [get_time]
sleep 1
set QeS [get_time]
sleep 1
#
# Start a long job (will be cancelled)
# Test job will depend on it to test non-eligible
#
set def_node [lindex [get_nodes_by_state] 0]
set jobid_dependent [submit_job -fail "-t5 -w $def_node --exclusive -J queue-$test_name --output=/dev/null --error=/dev/null --wrap '$bin_sleep 300'"]
wait_for_job -fail $jobid_dependent "RUNNING"
#
# Start a long job (will be cancelled)
# Test job will wait until this job, to test eligible
#
set def_node [lindex [get_nodes_by_state] 0]
set jobid_queue [submit_job -fail "-t5 -w $def_node --exclusive -J queue-$test_name --output=/dev/null --error=/dev/null --wrap '$bin_sleep 300'"]
#
# Start main job used for testing
#
set jobid [submit_job -fail "-t5 --dependency=afterany:$jobid_dependent -w $def_node -J $jobname-$test_name --output=/dev/null --error=/dev/null --wrap '$bin_sleep 300'"]
# Ensure that job is actually pending (non-eligible)
wait_for_job $jobid "PENDING"
# Wait for the submit time to be recorded in the accounting database
wait_for_job_acct -fail -timeout 30 $jobid submit
set QsE [get_time]
sleep 1
set QeE [get_time]
sleep 1
log_info "Test cases while the job is still NonEligible..."
testproc sacct_query $jobname "-j $jobid -s R" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeE" false
#
# Test -s PD
#
testproc sacct_query $jobname "-j $jobid -s PD" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeE" false
#
# With -j and not -s:
# -S defaults to Epoch 0
# -E defaults to Now
#
testproc sacct_query $jobname "-j $jobid" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeE" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeE" true
testproc sacct_query $jobname "-j $jobid -S $QsS" true
testproc sacct_query $jobname "-j $jobid -S $QsE" true
testproc sacct_query $jobname "-j $jobid -E $QeS" false
testproc sacct_query $jobname "-j $jobid -E $QeE" true
#
# With -s and not -j
# -S defaults to Now
# -E defaults to -S
#
testproc sacct_query $jobname "-s R" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeS" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeE" false
testproc sacct_query $jobname "-s R -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsE -E $QeE" false
testproc sacct_query $jobname "-s R -S $QsS" false
testproc sacct_query $jobname "-s R -S $QsE" false
testproc sacct_query $jobname "-s R -E $QeS" false true
testproc sacct_query $jobname "-s R -E $QeE" false true
# For -s PD
testproc sacct_query $jobname "-s PD" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeS" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeE" false
testproc sacct_query $jobname "-s PD -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeE" false
testproc sacct_query $jobname "-s PD -S $QsS" false
testproc sacct_query $jobname "-s PD -S $QsE" false
testproc sacct_query $jobname "-s PD -E $QeS" false true
testproc sacct_query $jobname "-s PD -E $QeE" false true
#
# Without none -s nor -j
# -S defaults to Mindnight
# -E defaults to Now
#
testproc sacct_query $jobname "" false
testproc sacct_query $jobname "-S $QsS -E $QeS" false
testproc sacct_query $jobname "-S $QsS -E $QeE" false
testproc sacct_query $jobname "-S $QsE -E $QeS" false true
testproc sacct_query $jobname "-S $QsE -E $QeE" false
testproc sacct_query $jobname "-S $QsS" false
testproc sacct_query $jobname "-S $QsE" false
testproc sacct_query $jobname "-E $QeS" false
testproc sacct_query $jobname "-E $QeE" false
# cancel the first job and wait until the second one starts running
# main job should be then eligible
cancel_job $jobid_dependent
wait_for_job $jobid_queue "RUNNING"
# Wait for the eligible time to be recorded in the accounting database
wait_for_job_acct -fail -timeout 30 $jobid eligible
set QsP [get_time]
sleep 1
set QeP [get_time]
sleep 1
#
# Test cases while the job is still pending
#
# With -j and -s:
# -S defaults to Epoch 0
# -E defaults to -S (unless no -S then Now)
#
log_info "Test cases while the job is still PENDING..."
testproc sacct_query $jobname "-j $jobid -s R" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsP" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeP" false
# Test -s PD
testproc sacct_query $jobname "-j $jobid -s PD" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP" true
testproc sacct_query $jobname "-j $jobid -s PD -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeP" true
#
# With -j and not -s:
# -S defaults to Epoch 0
# -E defaults to Now
#
testproc sacct_query $jobname "-j $jobid" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeE" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeE" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsS" true
testproc sacct_query $jobname "-j $jobid -S $QsE" true
testproc sacct_query $jobname "-j $jobid -S $QsP" true
testproc sacct_query $jobname "-j $jobid -E $QeS" false
testproc sacct_query $jobname "-j $jobid -E $QeE" true
testproc sacct_query $jobname "-j $jobid -E $QeP" true
#
# With -s and not -j
# -S defaults to Now
# -E defaults to -S
#
testproc sacct_query $jobname "-s R" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeS" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeE" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsE -E $QeE" false
testproc sacct_query $jobname "-s R -S $QsE -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-s R -S $QsP -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsS" false
testproc sacct_query $jobname "-s R -S $QsE" false
testproc sacct_query $jobname "-s R -S $QsP" false
testproc sacct_query $jobname "-s R -E $QeS" false true
testproc sacct_query $jobname "-s R -E $QeE" false true
testproc sacct_query $jobname "-s R -E $QeP" false true
# For -s PD
testproc sacct_query $jobname "-s PD" true
testproc sacct_query $jobname "-s PD -S $QsS -E $QeS" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeE" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeE" false
testproc sacct_query $jobname "-s PD -S $QsE -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsS" false
testproc sacct_query $jobname "-s PD -S $QsE" false
testproc sacct_query $jobname "-s PD -S $QsP" true
testproc sacct_query $jobname "-s PD -E $QeS" false true
testproc sacct_query $jobname "-s PD -E $QeE" false true
testproc sacct_query $jobname "-s PD -E $QeP" false true
#
# Without none -s nor -j
# -S defaults to Midnight
# -E defaults to Now
#
testproc sacct_query $jobname "" true
testproc sacct_query $jobname "-S $QsS -E $QeS" false
testproc sacct_query $jobname "-S $QsS -E $QeE" false
testproc sacct_query $jobname "-S $QsS -E $QeP" true
testproc sacct_query $jobname "-S $QsE -E $QeS" false true
testproc sacct_query $jobname "-S $QsE -E $QeE" false
testproc sacct_query $jobname "-S $QsE -E $QeP" true
testproc sacct_query $jobname "-S $QsP -E $QeS" false true
testproc sacct_query $jobname "-S $QsP -E $QeE" false true
testproc sacct_query $jobname "-S $QsP -E $QeP" true
testproc sacct_query $jobname "-S $QsS" true
testproc sacct_query $jobname "-S $QsE" true
testproc sacct_query $jobname "-S $QsP" true
testproc sacct_query $jobname "-E $QeS" false
testproc sacct_query $jobname "-E $QeE" false
testproc sacct_query $jobname "-E $QeP" true
#
# Canceling the blocking job and wait for the actual one to run
#
cancel_job $jobid_queue
wait_for_job $jobid "RUNNING"
# Wait for the start time to be recorded in the accounting database
wait_for_job_acct -fail -timeout 30 $jobid start
#
# Get query times while the job is running (QsR and QeR)
#
set QsR [get_time]
sleep 1
set QeR [get_time]
sleep 1
#
# Test cases while the job is still running
#
# With -j and -s:
# -S defaults to Epoch 0
# -E defaults to -S (unless no -S then Now)
#
log_info "Test cases while the job is still running..."
testproc sacct_query $jobname "-j $jobid -s R" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsS" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsR" true
testproc sacct_query $jobname "-j $jobid -s R -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeR" true
# Test -s PD
testproc sacct_query $jobname "-j $jobid -s PD" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeR" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -E $QeR" true
#
# With -j and not -s:
# -S defaults to Epoch 0
# -E defaults to Now
#
testproc sacct_query $jobname "-j $jobid" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeE" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeR" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeE" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeR" true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeR" true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeR" true
testproc sacct_query $jobname "-j $jobid -S $QsS" true
testproc sacct_query $jobname "-j $jobid -S $QsE" true
testproc sacct_query $jobname "-j $jobid -S $QsP" true
testproc sacct_query $jobname "-j $jobid -S $QsR" true
testproc sacct_query $jobname "-j $jobid -E $QeS" false
testproc sacct_query $jobname "-j $jobid -E $QeE" true
testproc sacct_query $jobname "-j $jobid -E $QeP" true
testproc sacct_query $jobname "-j $jobid -E $QeR" true
#
# With -s and not -j
# -S defaults to Now
# -E defaults to -S
#
testproc sacct_query $jobname "-s R" true
testproc sacct_query $jobname "-s R -S $QsS -E $QeS" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeE" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeR" true
testproc sacct_query $jobname "-s R -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsE -E $QeE" false
testproc sacct_query $jobname "-s R -S $QsE -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsE -E $QeR" true
testproc sacct_query $jobname "-s R -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-s R -S $QsP -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsP -E $QeR" true
testproc sacct_query $jobname "-s R -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-s R -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-s R -S $QsR -E $QeR" true
testproc sacct_query $jobname "-s R -S $QsS" false
testproc sacct_query $jobname "-s R -S $QsE" false
testproc sacct_query $jobname "-s R -S $QsP" false
testproc sacct_query $jobname "-s R -S $QsR" true
testproc sacct_query $jobname "-s R -E $QeS" false true
testproc sacct_query $jobname "-s R -E $QeE" false true
testproc sacct_query $jobname "-s R -E $QeP" false true
testproc sacct_query $jobname "-s R -E $QeR" false true
# For -s PD
testproc sacct_query $jobname "-s PD" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeS" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeE" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsS -E $QeR" true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeE" false
testproc sacct_query $jobname "-s PD -S $QsE -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeR" true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeR" true
testproc sacct_query $jobname "-s PD -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-s PD -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-s PD -S $QsR -E $QeR" false
testproc sacct_query $jobname "-s PD -S $QsS" false
testproc sacct_query $jobname "-s PD -S $QsE" false
testproc sacct_query $jobname "-s PD -S $QsP" true
testproc sacct_query $jobname "-s PD -S $QsR" false
testproc sacct_query $jobname "-s PD -E $QeS" false true
testproc sacct_query $jobname "-s PD -E $QeE" false true
testproc sacct_query $jobname "-s PD -E $QeP" false true
testproc sacct_query $jobname "-s PD -E $QeR" false true
#
# Without none -s nor -j
# -S defaults to Mindnight
# -E defaults to Now
#
testproc sacct_query $jobname "" true
testproc sacct_query $jobname "-S $QsS -E $QeS" false
testproc sacct_query $jobname "-S $QsS -E $QeE" false
testproc sacct_query $jobname "-S $QsS -E $QeP" true
testproc sacct_query $jobname "-S $QsS -E $QeR" true
testproc sacct_query $jobname "-S $QsE -E $QeS" false true
testproc sacct_query $jobname "-S $QsE -E $QeE" false
testproc sacct_query $jobname "-S $QsE -E $QeP" true
testproc sacct_query $jobname "-S $QsE -E $QeR" true
testproc sacct_query $jobname "-S $QsP -E $QeS" false true
testproc sacct_query $jobname "-S $QsP -E $QeE" false true
testproc sacct_query $jobname "-S $QsP -E $QeP" true
testproc sacct_query $jobname "-S $QsP -E $QeR" true
testproc sacct_query $jobname "-S $QsR -E $QeS" false true
testproc sacct_query $jobname "-S $QsR -E $QeE" false true
testproc sacct_query $jobname "-S $QsR -E $QeP" false true
testproc sacct_query $jobname "-S $QsR -E $QeR" true
testproc sacct_query $jobname "-S $QsS" true
testproc sacct_query $jobname "-S $QsE" true
testproc sacct_query $jobname "-S $QsP" true
testproc sacct_query $jobname "-S $QsR" true
testproc sacct_query $jobname "-E $QeS" false
testproc sacct_query $jobname "-E $QeE" false
testproc sacct_query $jobname "-E $QeP" true
testproc sacct_query $jobname "-E $QeR" true
#
# Cancel running job
#
cancel_job $jobid
# Wait for the end time to be recorded in the accounting database
wait_for_job_acct -fail -timeout 30 $jobid end
#
# Get query times when job is no more running
#
set QsC [get_time]
sleep 1
set QeC [get_time]
sleep 1
#
# Test cases for non-running job
#
# With -j and -s:
# -S defaults to Epoch 0
# -E defaults to -S (unless no -S then Now)
#
log_info "Test cases when the job is not running..."
testproc sacct_query $jobname "-j $jobid -s R" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsS -E $QeC" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsE -E $QeC" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsP -E $QeC" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsR -E $QeC" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsC -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsC -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsC -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsC -E $QeR" false true
testproc sacct_query $jobname "-j $jobid -s R -S $QsC -E $QeC" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsS" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsE" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsP" false
testproc sacct_query $jobname "-j $jobid -s R -S $QsR" true
testproc sacct_query $jobname "-j $jobid -s R -S $QsC" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeP" false
testproc sacct_query $jobname "-j $jobid -s R -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s R -E $QeC" true
# With -j and -s:
# -S defaults to Epoch 0
# -E defaults to -S (unless no -S then Now)
#
# For -s PD
testproc sacct_query $jobname "-j $jobid -s PD" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS -E $QeC" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE -E $QeC" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP -E $QeC" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeR" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR -E $QeC" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsC -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsC -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsC -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsC -E $QeR" false true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsC -E $QeC" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsS" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsE" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsP" true
testproc sacct_query $jobname "-j $jobid -s PD -S $QsR" false
testproc sacct_query $jobname "-j $jobid -s PD -S $QsC" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeS" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeE" false
testproc sacct_query $jobname "-j $jobid -s PD -E $QeP" true
testproc sacct_query $jobname "-j $jobid -s PD -E $QeR" true
testproc sacct_query $jobname "-j $jobid -s PD -E $QeC" true
#
# With -j and not -s:
# -S defaults to Epoch 0
# -E defaults to Now
#
testproc sacct_query $jobname "-j $jobid" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeS" false
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeE" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeR" true
testproc sacct_query $jobname "-j $jobid -S $QsS -E $QeC" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeE" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeR" true
testproc sacct_query $jobname "-j $jobid -S $QsE -E $QeC" true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeP" true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeR" true
testproc sacct_query $jobname "-j $jobid -S $QsP -E $QeC" true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeR" true
testproc sacct_query $jobname "-j $jobid -S $QsR -E $QeC" true
testproc sacct_query $jobname "-j $jobid -S $QsC -E $QeS" false true
testproc sacct_query $jobname "-j $jobid -S $QsC -E $QeE" false true
testproc sacct_query $jobname "-j $jobid -S $QsC -E $QeP" false true
testproc sacct_query $jobname "-j $jobid -S $QsC -E $QeR" false true
testproc sacct_query $jobname "-j $jobid -S $QsC -E $QeC" false
testproc sacct_query $jobname "-j $jobid -S $QsS" true
testproc sacct_query $jobname "-j $jobid -S $QsE" true
testproc sacct_query $jobname "-j $jobid -S $QsP" true
testproc sacct_query $jobname "-j $jobid -S $QsR" true
testproc sacct_query $jobname "-j $jobid -S $QsC" false
testproc sacct_query $jobname "-j $jobid -E $QeS" false
testproc sacct_query $jobname "-j $jobid -E $QeE" true
testproc sacct_query $jobname "-j $jobid -E $QeP" true
testproc sacct_query $jobname "-j $jobid -E $QeR" true
testproc sacct_query $jobname "-j $jobid -E $QeC" true
#
# With -s and not -j
# -S defaults to Now
# -E defaults to -S
#
testproc sacct_query $jobname "-s R" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeS" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeE" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsS -E $QeR" true
testproc sacct_query $jobname "-s R -S $QsS -E $QeC" true
testproc sacct_query $jobname "-s R -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsE -E $QeE" false
testproc sacct_query $jobname "-s R -S $QsE -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsE -E $QeR" true
testproc sacct_query $jobname "-s R -S $QsE -E $QeC" true
testproc sacct_query $jobname "-s R -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-s R -S $QsP -E $QeP" false
testproc sacct_query $jobname "-s R -S $QsP -E $QeR" true
testproc sacct_query $jobname "-s R -S $QsP -E $QeC" true
testproc sacct_query $jobname "-s R -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-s R -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-s R -S $QsR -E $QeR" true
testproc sacct_query $jobname "-s R -S $QsR -E $QeC" true
testproc sacct_query $jobname "-s R -S $QsC -E $QeS" false true
testproc sacct_query $jobname "-s R -S $QsC -E $QeE" false true
testproc sacct_query $jobname "-s R -S $QsC -E $QeP" false true
testproc sacct_query $jobname "-s R -S $QsC -E $QeR" false true
testproc sacct_query $jobname "-s R -S $QsC -E $QeC" false
testproc sacct_query $jobname "-s R -S $QsS" false
testproc sacct_query $jobname "-s R -S $QsE" false
testproc sacct_query $jobname "-s R -S $QsP" false
testproc sacct_query $jobname "-s R -S $QsR" true
testproc sacct_query $jobname "-s R -S $QsC" false
testproc sacct_query $jobname "-s R -E $QeS" false true
testproc sacct_query $jobname "-s R -E $QeE" false true
testproc sacct_query $jobname "-s R -E $QeP" false true
testproc sacct_query $jobname "-s R -E $QeR" false true
testproc sacct_query $jobname "-s R -E $QeC" false true
#
# With -s and not -j
# -S defaults to Now
# -E defaults to -S
#
# For -s PD
#
testproc sacct_query $jobname "-s PD" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeS" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeE" false
testproc sacct_query $jobname "-s PD -S $QsS -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsS -E $QeR" true
testproc sacct_query $jobname "-s PD -S $QsS -E $QeC" true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeE" false
testproc sacct_query $jobname "-s PD -S $QsE -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeR" true
testproc sacct_query $jobname "-s PD -S $QsE -E $QeC" true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeE" false true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeP" true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeR" true
testproc sacct_query $jobname "-s PD -S $QsP -E $QeC" true
testproc sacct_query $jobname "-s PD -S $QsR -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsR -E $QeE" false true
testproc sacct_query $jobname "-s PD -S $QsR -E $QeP" false true
testproc sacct_query $jobname "-s PD -S $QsR -E $QeR" false
testproc sacct_query $jobname "-s PD -S $QsR -E $QeC" false
testproc sacct_query $jobname "-s PD -S $QsC -E $QeS" false true
testproc sacct_query $jobname "-s PD -S $QsC -E $QeE" false true
testproc sacct_query $jobname "-s PD -S $QsC -E $QeP" false true
testproc sacct_query $jobname "-s PD -S $QsC -E $QeR" false true
testproc sacct_query $jobname "-s PD -S $QsC -E $QeC" false
testproc sacct_query $jobname "-s PD -S $QsS" false
testproc sacct_query $jobname "-s PD -S $QsE" false
testproc sacct_query $jobname "-s PD -S $QsP" true
testproc sacct_query $jobname "-s PD -S $QsR" false
testproc sacct_query $jobname "-s PD -S $QsC" false
testproc sacct_query $jobname "-s PD -E $QeS" false true
testproc sacct_query $jobname "-s PD -E $QeE" false true
testproc sacct_query $jobname "-s PD -E $QeP" false true
testproc sacct_query $jobname "-s PD -E $QeR" false true
testproc sacct_query $jobname "-s PD -E $QeC" false true
#
# Without none -s nor -j
# -S defaults to Midnight
# -E defaults to Now
#
testproc sacct_query $jobname "" true
testproc sacct_query $jobname "-S $QsS -E $QeS" false
testproc sacct_query $jobname "-S $QsS -E $QeE" false
testproc sacct_query $jobname "-S $QsS -E $QeP" true
testproc sacct_query $jobname "-S $QsS -E $QeR" true
testproc sacct_query $jobname "-S $QsS -E $QeC" true
testproc sacct_query $jobname "-S $QsE -E $QeS" false true
testproc sacct_query $jobname "-S $QsE -E $QeE" false
testproc sacct_query $jobname "-S $QsE -E $QeP" true
testproc sacct_query $jobname "-S $QsE -E $QeR" true
testproc sacct_query $jobname "-S $QsE -E $QeC" true
testproc sacct_query $jobname "-S $QsP -E $QeS" false true
testproc sacct_query $jobname "-S $QsP -E $QeE" false true
testproc sacct_query $jobname "-S $QsP -E $QeP" true
testproc sacct_query $jobname "-S $QsP -E $QeR" true
testproc sacct_query $jobname "-S $QsP -E $QeC" true
testproc sacct_query $jobname "-S $QsR -E $QeS" false true
testproc sacct_query $jobname "-S $QsR -E $QeE" false true
testproc sacct_query $jobname "-S $QsR -E $QeP" false true
testproc sacct_query $jobname "-S $QsR -E $QeR" true
testproc sacct_query $jobname "-S $QsR -E $QeC" true
testproc sacct_query $jobname "-S $QsC -E $QeS" false true
testproc sacct_query $jobname "-S $QsC -E $QeE" false true
testproc sacct_query $jobname "-S $QsC -E $QeP" false true
testproc sacct_query $jobname "-S $QsC -E $QeR" false true
testproc sacct_query $jobname "-S $QsC -E $QeC" false
testproc sacct_query $jobname "-S $QsS" true
testproc sacct_query $jobname "-S $QsE" true
testproc sacct_query $jobname "-S $QsP" true
testproc sacct_query $jobname "-S $QsR" true
testproc sacct_query $jobname "-S $QsC" false
testproc sacct_query $jobname "-E $QeS" false
testproc sacct_query $jobname "-E $QeE" false
testproc sacct_query $jobname "-E $QeP" true
testproc sacct_query $jobname "-E $QeR" true
testproc sacct_query $jobname "-E $QeC" true
log_info "Jobid : $jobid"
log_info "QsS : $QsS"
log_info "QeS : $QeS"
log_info "QsE : $QsE"
log_info "QeE : $QeE"
log_info "QsP : $QsP"
log_info "QeP : $QeP"
log_info "QsR : $QsR"
log_info "QeR : $QeR"
log_info "QsC : $QsC"
log_info "QeC : $QeC"