#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
#          Test sacctmgr update job [condition] set newwckey="newkey"
############################################################################
# Copyright (C) SchedMD LLC.
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals
source ./globals_accounting

# globals
set access_err    0

set sql_in_clus1  "$test_name-clus1-in.sql"
set sql_rem_clus1 "$test_name-clus1-rem.sql"
set sql_in_clus2  "$test_name-clus2-in.sql"
set sql_rem_clus2 "$test_name-clus2-rem.sql"

set cluster1      [format "%s%s" $test_name "clus1"]
set cluster2      [format "%s%s" $test_name "clus2"]
set account1      [format "%s%s" $test_name "acct1"]
set account2      [format "%s%s" $test_name "acct2"]
set user1         [get_my_user_name]
set user2         "root"
set wckey1        [format "%s%s" $test_name "wckey1"]
set wckey2        [format "%s%s" $test_name "wckey2"]
set wckey_new     [format "%s%s" $test_name "wckeynew"]

set clusters      [format "%s,%s" $cluster1 $cluster2]
set accounts      [format "%s,%s" $account1 $account2]
set users         [format "%s,%s" $user1    $user2]
set wckeys        [format "%s,%s" $wckey1   $wckey2]

# 8 jobs to test combinations of 2 accounts, 2 users and 2 wckeys
# same job ids will be used for the 2 clusters
set jobid01       "1001"
set jobid02       "1002"
set jobid03       "1003"
set jobid04       "1004"
set jobid05       "1005"
set jobid06       "1006"
set jobid07       "1007"
set jobid08       "1008"

###############################################################################
#
# Proc: Creates the sql files to load 8 test jobs, and to remove them.
#       Each job is a unique combination of 2 accounts, 2 users and 2 wckeys
#
# Input: cluster: cluster name to use
#        sql_in : filename to save the creation file
#        sql_rem: filename to save the remove file
#
# TODO: Error handling.
###############################################################################
proc create_sql {cluster sql_in sql_rem} {

	global bin_rm
	global test_name
	global account1 account2
	global wckey1 wckey2
	global user1acct1 user1acct2
	global user2acct1 user2acct2
	global user1wckey1 user1wckey2
	global user2wckey1 user2wckey2
	global jobid01 jobid02 jobid03 jobid04 jobid05 jobid06 jobid07 jobid08

	set node_list     [format "%s%s" $cluster "\[0-1\]"]
	set job_name      "$test_name\_job"
	set cluster_cpus  4

	set uid [get_my_uid]
	set gid [get_my_gid]

	# Mon Dec 31 23:00:00 2007
	set start_date "Mon Dec 31 23:00:00 2007"
	set period_start [exec date -d $start_date +%s]

	# Thu Jan 31 23:59:59 2008
	set end_date "Thu Jan 31 23:59:59 2008"
	set period_end [exec date -d $end_date +%s]

	# DON'T MESS WITH THIS UNLESS YOU REALLY UNDERSTAND WHAT YOU ARE DOING!!!!!
	# THIS COULD SERIOUSLY MESS UP YOUR DATABASE IF YOU ALTER THIS INCORRECTLY
	# JUST A FRIENDLY REMINDER ;)

	# create the in sql file
	exec $bin_rm -f $sql_in
	set file [open $sql_in "w"]

	# put in the cluster for back in the day before accounting was made here for us we are using 'Tue Jan  1 00:00:00 2008' = 1199174400 as the start
	puts $file "insert into cluster_event_table (node_name, cluster, tres, period_start, period_end, reason, cluster_nodes) values"
	puts $file "('', '$cluster', '1=$cluster_cpus', $period_start, $period_end, 'Cluster processor count', '$node_list' )"
	puts $file "on duplicate key update period_start=VALUES(period_start), period_end=VALUES(period_end);"

	# add jobs
	puts $file "insert into job_table (jobid, associd, wckey, wckeyid, uid, gid, `partition`, blockid, cluster, account, eligible, submit, start, end, suspended, name, state, comp_code, priority, req_cpus, tres_alloc, nodelist, kill_requid, qos, deleted) values"
	puts $file "('$jobid01', '$user1acct1', '$wckey1', '$user1wckey1', '$uid', '$gid', 'debug', '', '$cluster', '$account1', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid02', '$user1acct1', '$wckey2', '$user1wckey2', '$uid', '$gid', 'debug', '', '$cluster', '$account1', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid03', '$user2acct1', '$wckey1', '$user2wckey1',    '0',    '0', 'debug', '', '$cluster', '$account1', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid04', '$user2acct1', '$wckey2', '$user2wckey2',    '0',    '0', 'debug', '', '$cluster', '$account1', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"

	puts $file "('$jobid05', '$user1acct2', '$wckey1', '$user1wckey1', '$uid', '$gid', 'debug', '', '$cluster', '$account2', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid06', '$user1acct2', '$wckey2', '$user1wckey2', '$uid', '$gid', 'debug', '', '$cluster', '$account2', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid07', '$user2acct2', '$wckey1', '$user2wckey1',    '0',    '0', 'debug', '', '$cluster', '$account2', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid08', '$user2acct2', '$wckey2', '$user2wckey2',    '0',    '0', 'debug', '', '$cluster', '$account2', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "on duplicate key update id=LAST_INSERT_ID(id), eligible=VALUES(eligible), submit=VALUES(submit), start=VALUES(start), end=VALUES(end), associd=VALUES(associd), tres_alloc=VALUES(tres_alloc), wckey=VALUES(wckey), wckeyid=VALUES(wckeyid);";
	close $file

	# create the remove sql file
	exec $bin_rm -f $sql_rem
	set file [open $sql_rem "w"]

	puts $file [format "%s%s%s" "truncate table \"" $cluster "_event_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_job_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_step_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_usage_day_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_usage_hour_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_usage_month_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_assoc_usage_day_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_assoc_usage_hour_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_assoc_usage_month_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_wckey_usage_day_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_wckey_usage_hour_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_wckey_usage_month_table\";"]
	close $file
}


###############################################################################
#
# Proc: Runs the desired "sacctmgr update job" command and checked the expected
#       output.
#
# Returns: 0 if result was the expected, non-zero otherwise.
#
# Input: sacctmgr_args: the "where" args to the update job command.
#        expected_jobs: the list of job ids expected to be updated.
#        sql_rem: filename to save the remove file
#
###############################################################################
proc test_job_update {sacctmgr_args expected_jobs} {
	global sacct sacctmgr number
	global clusters wckey_new

	global sql_rem_clus1 sql_rem_clus2 sql_in_clus1 sql_in_clus2

	set rc 0
	incr rc [archive_load $sql_rem_clus1]
	incr rc [archive_load $sql_rem_clus2]
	incr rc [archive_load $sql_in_clus1]
	incr rc [archive_load $sql_in_clus2]
	if {$rc} {
		fail "Unable to load archive"
	}

	set matches_right 0
	set matches_wrong 0
	set output [run_command_output -fail "$sacctmgr -i update job set newwckey=$wckey_new where $sacctmgr_args"]
	set jobs_list [regexp -all -inline "($number) submitted at" $output]
	foreach {match job_id} $jobs_list {
		if {$job_id in $expected_jobs} {
			incr matches_right
		} else {
			incr matches_wrong
		}
	}
	subtest {$matches_right == [llength $expected_jobs] && $matches_wrong == 0} "Verify expected job updates in sacctmgr update for $sacctmgr_args" "($matches_right,$matches_wrong) != ([llength $expected_jobs],0)"

	set matches_right 0
	set matches_wrong 0
	set output [run_command_output -fail "$sacct -a -n --cluster=$clusters --wckey=$wckey_new --start=2000-01-01 --end=now --format=jobid"]
	set jobs_list [regexp -all -inline "($number)" $output]
	foreach {match job_id} $jobs_list {
		if {$job_id in $expected_jobs} {
			incr matches_right
		} else {
			incr matches_wrong
		}
	}
	subtest {$matches_right == [llength $expected_jobs] && $matches_wrong == 0} "Verify expected job updates in sacct for $sacctmgr_args" "($matches_right,$matches_wrong) != ([llength $expected_jobs],0)"
}

#
# Check configuration
# TODO: check also slurm.conf!!
#
if {[get_config_param "AccountingStorageType"] ne "accounting_storage/slurmdbd"} {
	skip "This test can't be run without a usable AccountStorageType"
}

if { [get_config_param -dbd "TrackWCKey"] eq "no" } {
	skip "This test can't be run without TrackWCKey set in slurmdbd.conf"
}

if {[get_admin_level] ne "Administrator"} {
	skip "This test can't be run without being an Accounting administrator.\nUse: sacctmgr mod user \$USER set admin=admin"
}

if {$user1 eq $user2} {
	skip "This test can't be run as root, because it needs 2 usernames and uses the current one and root"
}

###############################################################################
#
# Proc: Loads the remove sql entries and remove the added users, accounts and
#       clusters.
#
###############################################################################
proc cleanup {} {
	global code bin_rm
	global sql_in_clus1 sql_in_clus2
	global sql_rem_clus1 sql_rem_clus2
	global clusters accounts users

	if {[file exists $sql_rem_clus1] && [file exists $sql_rem_clus2]} {
		archive_load $sql_rem_clus1
		archive_load $sql_rem_clus2
	}

	remove_cluster $clusters
	remove_user    $clusters $accounts $users
	remove_acct    "" $accounts

	exec $bin_rm -f $sql_in_clus1 $sql_in_clus2 $sql_rem_clus1 $sql_rem_clus2
}


# Cluster Request
array set clus_req {}

# Account Request
array set acct_req {}
set acct_req(cluster) $clusters

# User Request
array set user_req {}
set user_req(cluster) $clusters
set user_req(account) $accounts
set user_req(wckey)   $wckeys

#
# Remove previous test clusters, accounts and users
#
# Make sure we have a clean system and permission to do this work
cleanup
if {$access_err != 0} {
	skip "Not authorized to perform this test"
}

# Add cluster
if [add_cluster $cluster1 [array get clus_req]] {
	fail "Unable to add cluster 1"
}
if [add_cluster $cluster2 [array get clus_req]] {
	fail "Unable to add cluster 2"
}

# Add accounts
if [add_acct "$account1,$account2" [array get acct_req]] {
	fail "Unable to add accounts"
}

# Add users
if [add_user $users [array get user_req]] {
	fail "Unable to add users"
}

#
# Get the association ids
# TODO: Assuming cluster1 and cluster2 has the same assoc ids
#
set user1acct1 0
set user1acct2 0
set user2acct1 0
set user2acct2 0
set output [run_command_output -fail "$sacctmgr -n -p list assoc users=$users account=$accounts cluster=$cluster1 format='User,account,id'"]
subtest -fatal {![regexp "There was a problem" $output]} "Verify that there's no error message from sacctmgr getting assocs"
subtest -fatal {[regexp "$user1.$account1.($number)." $output - user1acct1]} "Verify that user1acct1 is obtained"
subtest -fatal {[regexp "$user1.$account2.($number)." $output - user1acct2]} "Verify that user1acct2 is obtained"
subtest -fatal {[regexp "$user2.$account1.($number)." $output - user2acct1]} "Verify that user2acct1 is obtained"
subtest -fatal {[regexp "$user2.$account2.($number)." $output - user2acct2]} "Verify that user2acct2 is obtained"

#
# Get the wckeys ids
# TODO: Assuming cluster1 and cluster2 has the same wckeys ids
#
set user1wckey1 0
set user2wckey1 0
set user1wckey2 0
set user2wckey2 0
set output [run_command_output -fail "$sacctmgr -n -p list wckeys users=$users cluster=$clusters format='user,wckey,id'"]
subtest -fatal {![regexp "There was a problem" $output]} "Verify that there's no error message from sacctmgr getting wckeys"
subtest -fatal {[regexp "$user1.$wckey1.($number)." $output - user1wckey1]} "Verify that user1wckey1 is obtained"
subtest -fatal {[regexp "$user1.$wckey2.($number)." $output - user1wckey2]} "Verify that user1wckey2 is obtained"
subtest -fatal {[regexp "$user2.$wckey1.($number)." $output - user2wckey1]} "Verify that user2wckey1 is obtained"
subtest -fatal {[regexp "$user2.$wckey2.($number)." $output - user2wckey2]} "Verify that user2wckey2 is obtained"
#
# Create the SQL files
#
create_sql $cluster1 $sql_in_clus1 $sql_rem_clus1
create_sql $cluster2 $sql_in_clus2 $sql_rem_clus2

#
# Tests
#

# 1 cluster1, 1 user1
testproc test_job_update "cluster=$cluster1 user=$user1" [list $jobid01 $jobid02 $jobid05 $jobid06]

# 1 cluster1, 1 user2
testproc test_job_update "cluster=$cluster1 user=$user2" [list $jobid03 $jobid04 $jobid07 $jobid08]

# 1 cluster1, 2 users
testproc test_job_update "cluster=$cluster1 user=$user1,$user2" [list $jobid01 $jobid02 $jobid05 $jobid06 $jobid03 $jobid04 $jobid07 $jobid08]

# 1 cluster2, 1 user1
testproc test_job_update "cluster=$cluster2 user=$user1" [list $jobid01 $jobid02 $jobid05 $jobid06]

# 1 cluster2, 1 user2
testproc test_job_update "cluster=$cluster2 user=$user2" [list $jobid03 $jobid04 $jobid07 $jobid08]

# 1 cluster2, 2 users
testproc test_job_update "cluster=$cluster2 user=$user1,$user2" [list $jobid01 $jobid02 $jobid05 $jobid06 $jobid03 $jobid04 $jobid07 $jobid08]


# 1 cluster1, 1 account1, 1 user1
testproc test_job_update "cluster=$cluster1 account=$account1 user=$user1" [list $jobid01 $jobid02]

# 1 cluster1, 1 account1, 1 user2
testproc test_job_update "cluster=$cluster1 account=$account1 user=$user2" [list $jobid03 $jobid04]

# 1 cluster1, 1 account1, 2 users
testproc test_job_update "cluster=$cluster1 account=$account1 user=$user1,$user2" [list $jobid01 $jobid02 $jobid03 $jobid04]

# 1 cluster1, 1 account2, 1 user1
testproc test_job_update "cluster=$cluster1 account=$account2 user=$user1" [list $jobid05 $jobid06]

# 1 cluster1, 1 account2, 1 user2
testproc test_job_update "cluster=$cluster1 account=$account2 user=$user2" [list $jobid07 $jobid08]

# 1 cluster1, 1 account2, 2 users
testproc test_job_update "cluster=$cluster1 account=$account2 user=$user1,$user2" [list $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster1, 2 accounts, 1 user1
testproc test_job_update "cluster=$cluster1 account=$account1,$account2 user=$user1" [list $jobid01 $jobid02 $jobid05 $jobid06]

# 1 cluster1, 2 accounts, 1 user2
testproc test_job_update "cluster=$cluster1 account=$account1,$account2 user=$user2" [list $jobid03 $jobid04 $jobid07 $jobid08]

# 1 cluster1, 2 accounts, 2 users
testproc test_job_update "cluster=$cluster1 account=$account1,$account2 user=$user1,$user2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 1 cluster1, 1 wckey1, 1 user1
testproc test_job_update "cluster=$cluster1 wckey=$wckey1 user=$user1" [list $jobid01 $jobid05]

# 1 cluster1, 1 wckey1, 1 user2
testproc test_job_update "cluster=$cluster1 wckey=$wckey1 user=$user2" [list $jobid03 $jobid07]

# 1 cluster1, 1 wckey1, 2 users
testproc test_job_update "cluster=$cluster1 wckey=$wckey1 user=$user1,$user2" [list $jobid01 $jobid05 $jobid03 $jobid07]

# 1 cluster1, 1 wckey2, 1 user1
testproc test_job_update "cluster=$cluster1 wckey=$wckey2 user=$user1" [list $jobid02 $jobid06]

# 1 cluster1, 1 wckey2, 1 user2
testproc test_job_update "cluster=$cluster1 wckey=$wckey2 user=$user2" [list $jobid04 $jobid08]

# 1 cluster1, 1 wckey2, 2 users
testproc test_job_update "cluster=$cluster1 wckey=$wckey2 user=$user1,$user2" [list $jobid02 $jobid06 $jobid04 $jobid08]

# 1 cluster1, 2 wckeys, 1 user1
testproc test_job_update "cluster=$cluster1 wckey=$wckey1,$wckey2 user=$user1" [list $jobid01 $jobid02 $jobid05 $jobid06]

# 1 cluster1, 2 wckeys, 1 user2
testproc test_job_update "cluster=$cluster1 wckey=$wckey1,$wckey2 user=$user2" [list $jobid03 $jobid04 $jobid07 $jobid08]

# 1 cluster1, 2 wckeys, 2 users
testproc test_job_update "cluster=$cluster1 wckey=$wckey1,$wckey2 user=$user1,$user2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 1 cluster1, 1account1, 1 wckey1, 1 user1
testproc test_job_update "cluster=$cluster1 wckey=$wckey1 account=$account1 user=$user1" [list $jobid01]

# 1 cluster1, 1account1, 1 wckey1, 1 user2
testproc test_job_update "cluster=$cluster1 wckey=$wckey1 account=$account1 user=$user2" [list $jobid03]

# 1 cluster1, 1account1, 1 wckey1, 2 users
testproc test_job_update "cluster=$cluster1 wckey=$wckey1 account=$account1 user=$user1,$user2" [list $jobid01 $jobid03]

# 1 cluster1, 1account1, 1 wckey2, 1 user1
testproc test_job_update "cluster=$cluster1 wckey=$wckey2 account=$account1 user=$user1" [list $jobid02]

# 1 cluster1, 1account1, 1 wckey2, 1 user2
testproc test_job_update "cluster=$cluster1 wckey=$wckey2 account=$account1 user=$user2" [list $jobid04]

# 1 cluster1, 1account1, 1 wckey2, 2 users
testproc test_job_update "cluster=$cluster1 wckey=$wckey2 account=$account1 user=$user1,$user2" [list $jobid02 $jobid04]

# 1 cluster1, 1account1, 2 wckeys, 1 user1
testproc test_job_update "cluster=$cluster1 wckey=$wckey1,$wckey2 account=$account1 user=$user1" [list $jobid01 $jobid02]

# 1 cluster1, 1account1, 2 wckeys, 1 user2
testproc test_job_update "cluster=$cluster1 wckey=$wckey1,$wckey2 account=$account1 user=$user2" [list $jobid03 $jobid04]

# 1 cluster1, 1account1, 2 wckeys, 2 users
testproc test_job_update "cluster=$cluster1 wckey=$wckey1,$wckey2 account=$account1 user=$user1,$user2" [list $jobid01 $jobid02 $jobid03 $jobid04]


# 1 cluster1, 1 account1
testproc test_job_update "cluster=$cluster1 account=$account1" [list $jobid01 $jobid02 $jobid03 $jobid04]

# 1 cluster1, 1 account2
testproc test_job_update "cluster=$cluster1 account=$account2" [list $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster1, 2 accounts
testproc test_job_update "cluster=$cluster1 account=$account1,$account2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster2, 1 account1
testproc test_job_update "cluster=$cluster2 account=$account1" [list $jobid01 $jobid02 $jobid03 $jobid04]

# 1 cluster2, 1 account2
testproc test_job_update "cluster=$cluster2 account=$account2" [list $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster2, 2 accounts
testproc test_job_update "cluster=$cluster2 account=$account1,$account2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 1 cluster1, 1 wckey1
testproc test_job_update "cluster=$cluster1 wckey=$wckey1" [list $jobid01 $jobid03 $jobid05 $jobid07]

# 1 cluster1, 1 wckey2
testproc test_job_update "cluster=$cluster1 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08]

# 1 cluster1, 2 wckeys
testproc test_job_update "cluster=$cluster1 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster2, 1 wckey1
testproc test_job_update "cluster=$cluster2 wckey=$wckey1" [list $jobid01 $jobid03 $jobid05 $jobid07]

# 1 cluster2, 1 wckey2
testproc test_job_update "cluster=$cluster2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08]

# 1 cluster2, 2 wckeys
testproc test_job_update "cluster=$cluster2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 1 cluster1, 1 account1, 1 wckey1
testproc test_job_update "cluster=$cluster1 account=$account1 wckey=$wckey1" [list $jobid01 $jobid03]

# 1 cluster1, 1 account1, 1 wckey2
testproc test_job_update "cluster=$cluster1 account=$account1 wckey=$wckey2" [list $jobid02 $jobid04]

# 1 cluster1, 1 account1, 2 wckeys
testproc test_job_update "cluster=$cluster1 account=$account1 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04]

# 1 cluster1, 1 account2, 1 wckey1
testproc test_job_update "cluster=$cluster1 account=$account2 wckey=$wckey1" [list $jobid05 $jobid07]

# 1 cluster1, 1 account2, 1 wckey2
testproc test_job_update "cluster=$cluster1 account=$account2 wckey=$wckey2" [list $jobid06 $jobid08]

# 1 cluster1, 1 account2, 2 wckeys
testproc test_job_update "cluster=$cluster1 account=$account2 wckey=$wckey1,$wckey2" [list $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster1, 2 accounts, 1 wckey1
testproc test_job_update "cluster=$cluster1 account=$account1,$account2 wckey=$wckey1" [list  $jobid01 $jobid03 $jobid05 $jobid07]

# 1 cluster1, 2 accounts, 1 wckey2
testproc test_job_update "cluster=$cluster1 account=$account1,$account2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08]

# 1 cluster1, 2 accounts, 2 wckeys
testproc test_job_update "cluster=$cluster1 account=$account1,$account2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 1 cluster2, 1 account1, 1 wckey1
testproc test_job_update "cluster=$cluster2 account=$account1 wckey=$wckey1" [list $jobid01 $jobid03]

# 1 cluster2, 1 account1, 1 wckey2
testproc test_job_update "cluster=$cluster2 account=$account1 wckey=$wckey2" [list $jobid02 $jobid04]

# 1 cluster2, 1 account1, 2 wckeys
testproc test_job_update "cluster=$cluster2 account=$account1 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04]

# 1 cluster2, 1 account2, 1 wckey1
testproc test_job_update "cluster=$cluster2 account=$account2 wckey=$wckey1" [list $jobid05 $jobid07]

# 1 cluster2, 1 account2, 1 wckey2
testproc test_job_update "cluster=$cluster2 account=$account2 wckey=$wckey2" [list $jobid06 $jobid08]

# 1 cluster2, 1 account2, 2 wckeys
testproc test_job_update "cluster=$cluster2 account=$account2 wckey=$wckey1,$wckey2" [list $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster2, 2 accounts, 1 wckey1
testproc test_job_update "cluster=$cluster2 account=$account1,$account2 wckey=$wckey1" [list  $jobid01 $jobid03 $jobid05 $jobid07]

# 1 cluster2, 2 accounts, 1 wckey2
testproc test_job_update "cluster=$cluster2 account=$account1,$account2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08]

# 1 cluster2, 2 accounts, 2 wckeys
testproc test_job_update "cluster=$cluster2 account=$account1,$account2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 2 clusters, 1 account1
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account1" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid01 $jobid02 $jobid03 $jobid04]

# 2 clusters, 1 account2
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account2" [list $jobid05 $jobid06 $jobid07 $jobid08 $jobid05 $jobid06 $jobid07 $jobid08]

# 2 clusters, 2 accounts
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account1,$account2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08 $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]

# 2 clusters, 1 wckey1
testproc test_job_update "cluster=$cluster1,$cluster2 wckey=$wckey1" [list $jobid01 $jobid03 $jobid05 $jobid07 $jobid01 $jobid03 $jobid05 $jobid07]

# 2 clusters, 1 wckey2
testproc test_job_update "cluster=$cluster1,$cluster2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08 $jobid02 $jobid04 $jobid06 $jobid08]

# 2 clusters, 2 wckeys
testproc test_job_update "cluster=$cluster1,$cluster2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08 $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]

# 2 clusters, 1 account1, 1 wckey1
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account1 wckey=$wckey1" [list $jobid01 $jobid03 $jobid01 $jobid03]

# 2 clusters, 1 account1, 1 wckey2
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account1 wckey=$wckey2" [list $jobid02 $jobid04 $jobid02 $jobid04]

# 2 clusters, 1 account1, 2 wckeys
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account1 wckey=$wckey1,$wckey2" [list $jobid01 $jobid03 $jobid01 $jobid03 $jobid02 $jobid04 $jobid02 $jobid04]

# 2 clusters, 1 account2, 1 wckey1
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account2 wckey=$wckey1" [list $jobid05 $jobid07 $jobid05 $jobid07]

# 2 clusters, 1 account2, 1 wckey2
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account2 wckey=$wckey2" [list $jobid06 $jobid08 $jobid06 $jobid08]

# 2 clusters, 1 account2, 2 wckeys
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account2 wckey=$wckey1,$wckey2" [list $jobid05 $jobid07 $jobid05 $jobid07 $jobid06 $jobid08 $jobid06 $jobid08]

# 2 clusters, 2 accounts, 1 wckey1
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account1,$account2 wckey=$wckey1" [list $jobid01 $jobid03 $jobid01 $jobid03 $jobid05 $jobid07 $jobid05 $jobid07]

# 2 clusters, 2 accounts, 1 wckey2
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account1,$account2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid02 $jobid04 $jobid06 $jobid08 $jobid06 $jobid08]

# 2 clusters, 2 accounts, 2 wckeys
testproc test_job_update "cluster=$cluster1,$cluster2 account=$account1,$account2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid03 $jobid01 $jobid03 $jobid05 $jobid07 $jobid05 $jobid07 $jobid02 $jobid04 $jobid02 $jobid04 $jobid06 $jobid08 $jobid06 $jobid08]