blob: 4e53e167183bd35e818c3b227fdd8bda556ec022 [file] [log] [blame]
#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
# Test sacctmgr update job [condition] set newwckey="newkey"
############################################################################
# Copyright (C) SchedMD LLC.
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
############################################################################
source ./globals
source ./globals_accounting
# globals
set access_err 0
set sql_in_clus1 "$test_name-clus1-in.sql"
set sql_rem_clus1 "$test_name-clus1-rem.sql"
set sql_in_clus2 "$test_name-clus2-in.sql"
set sql_rem_clus2 "$test_name-clus2-rem.sql"
set cluster1 [format "%s%s" $test_name "clus1"]
set cluster2 [format "%s%s" $test_name "clus2"]
set account1 [format "%s%s" $test_name "acct1"]
set account2 [format "%s%s" $test_name "acct2"]
set user1 [get_my_user_name]
set user2 "root"
set wckey1 [format "%s%s" $test_name "wckey1"]
set wckey2 [format "%s%s" $test_name "wckey2"]
set wckey_new [format "%s%s" $test_name "wckeynew"]
set clusters [format "%s,%s" $cluster1 $cluster2]
set accounts [format "%s,%s" $account1 $account2]
set users [format "%s,%s" $user1 $user2]
set wckeys [format "%s,%s" $wckey1 $wckey2]
# 8 jobs to test combinations of 2 accounts, 2 users and 2 wckeys
# same job ids will be used for the 2 clusters
set jobid01 "1001"
set jobid02 "1002"
set jobid03 "1003"
set jobid04 "1004"
set jobid05 "1005"
set jobid06 "1006"
set jobid07 "1007"
set jobid08 "1008"
###############################################################################
#
# Proc: Creates the sql files to load 8 test jobs, and to remove them.
# Each job is a unique combination of 2 accounts, 2 users and 2 wckeys
#
# Input: cluster: cluster name to use
# sql_in : filename to save the creation file
# sql_rem: filename to save the remove file
#
# TODO: Error handling.
###############################################################################
proc create_sql {cluster sql_in sql_rem} {
global bin_rm
global test_name
global account1 account2
global wckey1 wckey2
global user1acct1 user1acct2
global user2acct1 user2acct2
global user1wckey1 user1wckey2
global user2wckey1 user2wckey2
global jobid01 jobid02 jobid03 jobid04 jobid05 jobid06 jobid07 jobid08
set node_list [format "%s%s" $cluster "\[0-1\]"]
set job_name "$test_name\_job"
set cluster_cpus 4
set uid [get_my_uid]
set gid [get_my_gid]
# Mon Dec 31 23:00:00 2007
set start_date "Mon Dec 31 23:00:00 2007"
set period_start [exec date -d $start_date +%s]
# Thu Jan 31 23:59:59 2008
set end_date "Thu Jan 31 23:59:59 2008"
set period_end [exec date -d $end_date +%s]
# DON'T MESS WITH THIS UNLESS YOU REALLY UNDERSTAND WHAT YOU ARE DOING!!!!!
# THIS COULD SERIOUSLY MESS UP YOUR DATABASE IF YOU ALTER THIS INCORRECTLY
# JUST A FRIENDLY REMINDER ;)
# create the in sql file
exec $bin_rm -f $sql_in
set file [open $sql_in "w"]
# put in the cluster for back in the day before accounting was made here for us we are using 'Tue Jan 1 00:00:00 2008' = 1199174400 as the start
puts $file "insert into cluster_event_table (node_name, cluster, tres, period_start, period_end, reason, cluster_nodes) values"
puts $file "('', '$cluster', '1=$cluster_cpus', $period_start, $period_end, 'Cluster processor count', '$node_list' )"
puts $file "on duplicate key update period_start=VALUES(period_start), period_end=VALUES(period_end);"
# add jobs
puts $file "insert into job_table (jobid, associd, wckey, wckeyid, uid, gid, `partition`, blockid, cluster, account, eligible, submit, start, end, suspended, name, state, comp_code, priority, req_cpus, tres_alloc, nodelist, kill_requid, qos, deleted) values"
puts $file "('$jobid01', '$user1acct1', '$wckey1', '$user1wckey1', '$uid', '$gid', 'debug', '', '$cluster', '$account1', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
puts $file "('$jobid02', '$user1acct1', '$wckey2', '$user1wckey2', '$uid', '$gid', 'debug', '', '$cluster', '$account1', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
puts $file "('$jobid03', '$user2acct1', '$wckey1', '$user2wckey1', '0', '0', 'debug', '', '$cluster', '$account1', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
puts $file "('$jobid04', '$user2acct1', '$wckey2', '$user2wckey2', '0', '0', 'debug', '', '$cluster', '$account1', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
puts $file "('$jobid05', '$user1acct2', '$wckey1', '$user1wckey1', '$uid', '$gid', 'debug', '', '$cluster', '$account2', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
puts $file "('$jobid06', '$user1acct2', '$wckey2', '$user1wckey2', '$uid', '$gid', 'debug', '', '$cluster', '$account2', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
puts $file "('$jobid07', '$user2acct2', '$wckey1', '$user2wckey1', '0', '0', 'debug', '', '$cluster', '$account2', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
puts $file "('$jobid08', '$user2acct2', '$wckey2', '$user2wckey2', '0', '0', 'debug', '', '$cluster', '$account2', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
puts $file "on duplicate key update id=LAST_INSERT_ID(id), eligible=VALUES(eligible), submit=VALUES(submit), start=VALUES(start), end=VALUES(end), associd=VALUES(associd), tres_alloc=VALUES(tres_alloc), wckey=VALUES(wckey), wckeyid=VALUES(wckeyid);";
close $file
# create the remove sql file
exec $bin_rm -f $sql_rem
set file [open $sql_rem "w"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_event_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_job_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_step_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_usage_day_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_usage_hour_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_usage_month_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_assoc_usage_day_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_assoc_usage_hour_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_assoc_usage_month_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_wckey_usage_day_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_wckey_usage_hour_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_wckey_usage_month_table\";"]
close $file
}
###############################################################################
#
# Proc: Runs the desired "sacctmgr update job" command and checked the expected
# output.
#
# Returns: 0 if result was the expected, non-zero otherwise.
#
# Input: sacctmgr_args: the "where" args to the update job command.
# expected_jobs: the list of job ids expected to be updated.
# sql_rem: filename to save the remove file
#
###############################################################################
proc test_job_update {sacctmgr_args expected_jobs} {
global sacct sacctmgr number
global clusters wckey_new
global sql_rem_clus1 sql_rem_clus2 sql_in_clus1 sql_in_clus2
set rc 0
incr rc [archive_load $sql_rem_clus1]
incr rc [archive_load $sql_rem_clus2]
incr rc [archive_load $sql_in_clus1]
incr rc [archive_load $sql_in_clus2]
if {$rc} {
fail "Unable to load archive"
}
set matches_right 0
set matches_wrong 0
set output [run_command_output -fail "$sacctmgr -i update job set newwckey=$wckey_new where $sacctmgr_args"]
set jobs_list [regexp -all -inline "($number) submitted at" $output]
foreach {match job_id} $jobs_list {
if {$job_id in $expected_jobs} {
incr matches_right
} else {
incr matches_wrong
}
}
subtest {$matches_right == [llength $expected_jobs] && $matches_wrong == 0} "Verify expected job updates in sacctmgr update for $sacctmgr_args" "($matches_right,$matches_wrong) != ([llength $expected_jobs],0)"
set matches_right 0
set matches_wrong 0
set output [run_command_output -fail "$sacct -a -n --cluster=$clusters --wckey=$wckey_new --start=2000-01-01 --end=now --format=jobid"]
set jobs_list [regexp -all -inline "($number)" $output]
foreach {match job_id} $jobs_list {
if {$job_id in $expected_jobs} {
incr matches_right
} else {
incr matches_wrong
}
}
subtest {$matches_right == [llength $expected_jobs] && $matches_wrong == 0} "Verify expected job updates in sacct for $sacctmgr_args" "($matches_right,$matches_wrong) != ([llength $expected_jobs],0)"
}
#
# Check configuration
# TODO: check also slurm.conf!!
#
if {[get_config_param "AccountingStorageType"] ne "accounting_storage/slurmdbd"} {
skip "This test can't be run without a usable AccountStorageType"
}
if { [get_config_param -dbd "TrackWCKey"] eq "no" } {
skip "This test can't be run without TrackWCKey set in slurmdbd.conf"
}
if {[get_admin_level] ne "Administrator"} {
skip "This test can't be run without being an Accounting administrator.\nUse: sacctmgr mod user \$USER set admin=admin"
}
if {$user1 eq $user2} {
skip "This test can't be run as root, because it needs 2 usernames and uses the current one and root"
}
###############################################################################
#
# Proc: Loads the remove sql entries and remove the added users, accounts and
# clusters.
#
###############################################################################
proc cleanup {} {
global code bin_rm
global sql_in_clus1 sql_in_clus2
global sql_rem_clus1 sql_rem_clus2
global clusters accounts users
if {[file exists $sql_rem_clus1] && [file exists $sql_rem_clus2]} {
archive_load $sql_rem_clus1
archive_load $sql_rem_clus2
}
remove_cluster $clusters
remove_user $clusters $accounts $users
remove_acct "" $accounts
exec $bin_rm -f $sql_in_clus1 $sql_in_clus2 $sql_rem_clus1 $sql_rem_clus2
}
# Cluster Request
array set clus_req {}
# Account Request
array set acct_req {}
set acct_req(cluster) $clusters
# User Request
array set user_req {}
set user_req(cluster) $clusters
set user_req(account) $accounts
set user_req(wckey) $wckeys
#
# Remove previous test clusters, accounts and users
#
# Make sure we have a clean system and permission to do this work
cleanup
if {$access_err != 0} {
skip "Not authorized to perform this test"
}
# Add cluster
if [add_cluster $cluster1 [array get clus_req]] {
fail "Unable to add cluster 1"
}
if [add_cluster $cluster2 [array get clus_req]] {
fail "Unable to add cluster 2"
}
# Add accounts
if [add_acct "$account1,$account2" [array get acct_req]] {
fail "Unable to add accounts"
}
# Add users
if [add_user $users [array get user_req]] {
fail "Unable to add users"
}
#
# Get the association ids
# TODO: Assuming cluster1 and cluster2 has the same assoc ids
#
set user1acct1 0
set user1acct2 0
set user2acct1 0
set user2acct2 0
set output [run_command_output -fail "$sacctmgr -n -p list assoc users=$users account=$accounts cluster=$cluster1 format='User,account,id'"]
subtest -fatal {![regexp "There was a problem" $output]} "Verify that there's no error message from sacctmgr getting assocs"
subtest -fatal {[regexp "$user1.$account1.($number)." $output - user1acct1]} "Verify that user1acct1 is obtained"
subtest -fatal {[regexp "$user1.$account2.($number)." $output - user1acct2]} "Verify that user1acct2 is obtained"
subtest -fatal {[regexp "$user2.$account1.($number)." $output - user2acct1]} "Verify that user2acct1 is obtained"
subtest -fatal {[regexp "$user2.$account2.($number)." $output - user2acct2]} "Verify that user2acct2 is obtained"
#
# Get the wckeys ids
# TODO: Assuming cluster1 and cluster2 has the same wckeys ids
#
set user1wckey1 0
set user2wckey1 0
set user1wckey2 0
set user2wckey2 0
set output [run_command_output -fail "$sacctmgr -n -p list wckeys users=$users cluster=$clusters format='user,wckey,id'"]
subtest -fatal {![regexp "There was a problem" $output]} "Verify that there's no error message from sacctmgr getting wckeys"
subtest -fatal {[regexp "$user1.$wckey1.($number)." $output - user1wckey1]} "Verify that user1wckey1 is obtained"
subtest -fatal {[regexp "$user1.$wckey2.($number)." $output - user1wckey2]} "Verify that user1wckey2 is obtained"
subtest -fatal {[regexp "$user2.$wckey1.($number)." $output - user2wckey1]} "Verify that user2wckey1 is obtained"
subtest -fatal {[regexp "$user2.$wckey2.($number)." $output - user2wckey2]} "Verify that user2wckey2 is obtained"
#
# Create the SQL files
#
create_sql $cluster1 $sql_in_clus1 $sql_rem_clus1
create_sql $cluster2 $sql_in_clus2 $sql_rem_clus2
#
# Tests
#
# 1 cluster1, 1 user1
testproc test_job_update "cluster=$cluster1 user=$user1" [list $jobid01 $jobid02 $jobid05 $jobid06]
# 1 cluster1, 1 user2
testproc test_job_update "cluster=$cluster1 user=$user2" [list $jobid03 $jobid04 $jobid07 $jobid08]
# 1 cluster1, 2 users
testproc test_job_update "cluster=$cluster1 user=$user1,$user2" [list $jobid01 $jobid02 $jobid05 $jobid06 $jobid03 $jobid04 $jobid07 $jobid08]
# 1 cluster2, 1 user1
testproc test_job_update "cluster=$cluster2 user=$user1" [list $jobid01 $jobid02 $jobid05 $jobid06]
# 1 cluster2, 1 user2
testproc test_job_update "cluster=$cluster2 user=$user2" [list $jobid03 $jobid04 $jobid07 $jobid08]
# 1 cluster2, 2 users
testproc test_job_update "cluster=$cluster2 user=$user1,$user2" [list $jobid01 $jobid02 $jobid05 $jobid06 $jobid03 $jobid04 $jobid07 $jobid08]
# 1 cluster1, 1 account1, 1 user1
testproc test_job_update "cluster=$cluster1 account=$account1 user=$user1" [list $jobid01 $jobid02]
# 1 cluster1, 1 account1, 1 user2
testproc test_job_update "cluster=$cluster1 account=$account1 user=$user2" [list $jobid03 $jobid04]
# 1 cluster1, 1 account1, 2 users
testproc test_job_update "cluster=$cluster1 account=$account1 user=$user1,$user2" [list $jobid01 $jobid02 $jobid03 $jobid04]
# 1 cluster1, 1 account2, 1 user1
testproc test_job_update "cluster=$cluster1 account=$account2 user=$user1" [list $jobid05 $jobid06]
# 1 cluster1, 1 account2, 1 user2
testproc test_job_update "cluster=$cluster1 account=$account2 user=$user2" [list $jobid07 $jobid08]
# 1 cluster1, 1 account2, 2 users
testproc test_job_update "cluster=$cluster1 account=$account2 user=$user1,$user2" [list $jobid05 $jobid06 $jobid07 $jobid08]
# 1 cluster1, 2 accounts, 1 user1
testproc test_job_update "cluster=$cluster1 account=$account1,$account2 user=$user1" [list $jobid01 $jobid02 $jobid05 $jobid06]
# 1 cluster1, 2 accounts, 1 user2
testproc test_job_update "cluster=$cluster1 account=$account1,$account2 user=$user2" [list $jobid03 $jobid04 $jobid07 $jobid08]
# 1 cluster1, 2 accounts, 2 users
testproc test_job_update "cluster=$cluster1 account=$account1,$account2 user=$user1,$user2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]
# 1 cluster1, 1 wckey1, 1 user1
testproc test_job_update "cluster=$cluster1 wckey=$wckey1 user=$user1" [list $jobid01 $jobid05]
# 1 cluster1, 1 wckey1, 1 user2
testproc test_job_update "cluster=$cluster1 wckey=$wckey1 user=$user2" [list $jobid03 $jobid07]
# 1 cluster1, 1 wckey1, 2 users
testproc test_job_update "cluster=$cluster1 wckey=$wckey1 user=$user1,$user2" [list $jobid01 $jobid05 $jobid03 $jobid07]
# 1 cluster1, 1 wckey2, 1 user1
testproc test_job_update "cluster=$cluster1 wckey=$wckey2 user=$user1" [list $jobid02 $jobid06]
# 1 cluster1, 1 wckey2, 1 user2
testproc test_job_update "cluster=$cluster1 wckey=$wckey2 user=$user2" [list $jobid04 $jobid08]
# 1 cluster1, 1 wckey2, 2 users
testproc test_job_update "cluster=$cluster1 wckey=$wckey2 user=$user1,$user2" [list $jobid02 $jobid06 $jobid04 $jobid08]
# 1 cluster1, 2 wckeys, 1 user1
testproc test_job_update "cluster=$cluster1 wckey=$wckey1,$wckey2 user=$user1" [list $jobid01 $jobid02 $jobid05 $jobid06]
# 1 cluster1, 2 wckeys, 1 user2
testproc test_job_update "cluster=$cluster1 wckey=$wckey1,$wckey2 user=$user2" [list $jobid03 $jobid04 $jobid07 $jobid08]
# 1 cluster1, 2 wckeys, 2 users
testproc test_job_update "cluster=$cluster1 wckey=$wckey1,$wckey2 user=$user1,$user2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]
# 1 cluster1, 1account1, 1 wckey1, 1 user1
testproc test_job_update "cluster=$cluster1 wckey=$wckey1 account=$account1 user=$user1" [list $jobid01]
# 1 cluster1, 1account1, 1 wckey1, 1 user2
testproc test_job_update "cluster=$cluster1 wckey=$wckey1 account=$account1 user=$user2" [list $jobid03]
# 1 cluster1, 1account1, 1 wckey1, 2 users
testproc test_job_update "cluster=$cluster1 wckey=$wckey1 account=$account1 user=$user1,$user2" [list $jobid01 $jobid03]
# 1 cluster1, 1account1, 1 wckey2, 1 user1
testproc test_job_update "cluster=$cluster1 wckey=$wckey2 account=$account1 user=$user1" [list $jobid02]
# 1 cluster1, 1account1, 1 wckey2, 1 user2
testproc test_job_update "cluster=$cluster1 wckey=$wckey2 account=$account1 user=$user2" [list $jobid04]
# 1 cluster1, 1account1, 1 wckey2, 2 users
testproc test_job_update "cluster=$cluster1 wckey=$wckey2 account=$account1 user=$user1,$user2" [list $jobid02 $jobid04]
# 1 cluster1, 1account1, 2 wckeys, 1 user1
testproc test_job_update "cluster=$cluster1 wckey=$wckey1,$wckey2 account=$account1 user=$user1" [list $jobid01 $jobid02]
# 1 cluster1, 1account1, 2 wckeys, 1 user2
testproc test_job_update "cluster=$cluster1 wckey=$wckey1,$wckey2 account=$account1 user=$user2" [list $jobid03 $jobid04]
# 1 cluster1, 1account1, 2 wckeys, 2 users
testproc test_job_update "cluster=$cluster1 wckey=$wckey1,$wckey2 account=$account1 user=$user1,$user2" [list $jobid01 $jobid02 $jobid03 $jobid04]
# 1 cluster1, 1 account1
testproc test_job_update "cluster=$cluster1 account=$account1" [list $jobid01 $jobid02 $jobid03 $jobid04]
# 1 cluster1, 1 account2
testproc test_job_update "cluster=$cluster1 account=$account2" [list $jobid05 $jobid06 $jobid07 $jobid08]
# 1 cluster1, 2 accounts
testproc test_job_update "cluster=$cluster1 account=$account1,$account2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]
# 1 cluster2, 1 account1
testproc test_job_update "cluster=$cluster2 account=$account1" [list $jobid01 $jobid02 $jobid03 $jobid04]
# 1 cluster2, 1 account2
testproc test_job_update "cluster=$cluster2 account=$account2" [list $jobid05 $jobid06 $jobid07 $jobid08]
# 1 cluster2, 2 accounts
testproc test_job_update "cluster=$cluster2 account=$account1,$account2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]
# 1 cluster1, 1 wckey1
testproc test_job_update "cluster=$cluster1 wckey=$wckey1" [list $jobid01 $jobid03 $jobid05 $jobid07]
# 1 cluster1, 1 wckey2
testproc test_job_update "cluster=$cluster1 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08]
# 1 cluster1, 2 wckeys
testproc test_job_update "cluster=$cluster1 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]
# 1 cluster2, 1 wckey1
testproc test_job_update "cluster=$cluster2 wckey=$wckey1" [list $jobid01 $jobid03 $jobid05 $jobid07]
# 1 cluster2, 1 wckey2
testproc test_job_update "cluster=$cluster2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08]
# 1 cluster2, 2 wckeys
testproc test_job_update "cluster=$cluster2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]
# 1 cluster1, 1 account1, 1 wckey1
testproc test_job_update "cluster=$cluster1 account=$account1 wckey=$wckey1" [list $jobid01 $jobid03]
# 1 cluster1, 1 account1, 1 wckey2
testproc test_job_update "cluster=$cluster1 account=$account1 wckey=$wckey2" [list $jobid02 $jobid04]
# 1 cluster1, 1 account1, 2 wckeys
testproc test_job_update "cluster=$cluster1 account=$account1 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04]
# 1 cluster1, 1 account2, 1 wckey1
testproc test_job_update "cluster=$cluster1 account=$account2 wckey=$wckey1" [list $jobid05 $jobid07]
# 1 cluster1, 1 account2, 1 wckey2
testproc test_job_update "cluster=$cluster1 account=$account2 wckey=$wckey2" [list $jobid06 $jobid08]
# 1 cluster1, 1 account2, 2 wckeys
testproc test_job_update "cluster=$cluster1 account=$account2 wckey=$wckey1,$wckey2" [list $jobid05 $jobid06 $jobid07 $jobid08]
# 1 cluster1, 2 accounts, 1 wckey1
testproc test_job_update "cluster=$cluster1 account=$account1,$account2 wckey=$wckey1" [list $jobid01 $jobid03 $jobid05 $jobid07]
# 1 cluster1, 2 accounts, 1 wckey2
testproc test_job_update "cluster=$cluster1 account=$account1,$account2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08]
# 1 cluster1, 2 accounts, 2 wckeys
testproc test_job_update "cluster=$cluster1 account=$account1,$account2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]
# 1 cluster2, 1 account1, 1 wckey1
testproc test_job_update "cluster=$cluster2 account=$account1 wckey=$wckey1" [list $jobid01 $jobid03]
# 1 cluster2, 1 account1, 1 wckey2
testproc test_job_update "cluster=$cluster2 account=$account1 wckey=$wckey2" [list $jobid02 $jobid04]
# 1 cluster2, 1 account1, 2 wckeys
testproc test_job_update "cluster=$cluster2 account=$account1 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04]
# 1 cluster2, 1 account2, 1 wckey1
testproc test_job_update "cluster=$cluster2 account=$account2 wckey=$wckey1" [list $jobid05 $jobid07]
# 1 cluster2, 1 account2, 1 wckey2
testproc test_job_update "cluster=$cluster2 account=$account2 wckey=$wckey2" [list $jobid06 $jobid08]
# 1 cluster2, 1 account2, 2 wckeys
testproc test_job_update "cluster=$cluster2 account=$account2 wckey=$wckey1,$wckey2" [list $jobid05 $jobid06 $jobid07 $jobid08]
# 1 cluster2, 2 accounts, 1 wckey1
testproc test_job_update "cluster=$cluster2 account=$account1,$account2 wckey=$wckey1" [list $jobid01 $jobid03 $jobid05 $jobid07]
# 1 cluster2, 2 accounts, 1 wckey2
testproc test_job_update "cluster=$cluster2 account=$account1,$account2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08]
# 1 cluster2, 2 accounts, 2 wckeys
testproc test_job_update "cluster=$cluster2 account=$account1,$account2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]
# 2 clusters, 1 account1
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account1" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid01 $jobid02 $jobid03 $jobid04]
# 2 clusters, 1 account2
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account2" [list $jobid05 $jobid06 $jobid07 $jobid08 $jobid05 $jobid06 $jobid07 $jobid08]
# 2 clusters, 2 accounts
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account1,$account2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08 $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]
# 2 clusters, 1 wckey1
testproc test_job_update "cluster=$cluster1,$cluster2 wckey=$wckey1" [list $jobid01 $jobid03 $jobid05 $jobid07 $jobid01 $jobid03 $jobid05 $jobid07]
# 2 clusters, 1 wckey2
testproc test_job_update "cluster=$cluster1,$cluster2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08 $jobid02 $jobid04 $jobid06 $jobid08]
# 2 clusters, 2 wckeys
testproc test_job_update "cluster=$cluster1,$cluster2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08 $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]
# 2 clusters, 1 account1, 1 wckey1
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account1 wckey=$wckey1" [list $jobid01 $jobid03 $jobid01 $jobid03]
# 2 clusters, 1 account1, 1 wckey2
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account1 wckey=$wckey2" [list $jobid02 $jobid04 $jobid02 $jobid04]
# 2 clusters, 1 account1, 2 wckeys
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account1 wckey=$wckey1,$wckey2" [list $jobid01 $jobid03 $jobid01 $jobid03 $jobid02 $jobid04 $jobid02 $jobid04]
# 2 clusters, 1 account2, 1 wckey1
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account2 wckey=$wckey1" [list $jobid05 $jobid07 $jobid05 $jobid07]
# 2 clusters, 1 account2, 1 wckey2
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account2 wckey=$wckey2" [list $jobid06 $jobid08 $jobid06 $jobid08]
# 2 clusters, 1 account2, 2 wckeys
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account2 wckey=$wckey1,$wckey2" [list $jobid05 $jobid07 $jobid05 $jobid07 $jobid06 $jobid08 $jobid06 $jobid08]
# 2 clusters, 2 accounts, 1 wckey1
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account1,$account2 wckey=$wckey1" [list $jobid01 $jobid03 $jobid01 $jobid03 $jobid05 $jobid07 $jobid05 $jobid07]
# 2 clusters, 2 accounts, 1 wckey2
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account1,$account2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid02 $jobid04 $jobid06 $jobid08 $jobid06 $jobid08]
# 2 clusters, 2 accounts, 2 wckeys
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account1,$account2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid03 $jobid01 $jobid03 $jobid05 $jobid07 $jobid05 $jobid07 $jobid02 $jobid04 $jobid02 $jobid04 $jobid06 $jobid08 $jobid06 $jobid08]