#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
#          to be called from test21.34
#          Tests partition and job qos limits
############################################################################
# Copyright (C) SchedMD LLC.
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./inc21.30.1
source ./inc21.30.2
source ./inc21.30.3
source ./inc21.30.4
source ./inc21.30.5
source ./inc21.30.6
source ./inc21.30.7
source ./inc21.30.8
source ./inc21.34.1
source ./inc21.30.10
source ./inc21.30.11
source ./inc21.34.2
source ./inc21.30.13
source ./inc21.30.14
source ./inc21.30.15
source ./inc21.30.16
source ./inc21.30.17

##########################################################
#
# Test that partition qos limits are enforced
#
##########################################################
proc part_test { } {

	global grn_num grcpu_num grjobs_num grsub_num grsub_num maxcpu_num
	global maxnode_num maxjobs_num maxjobsub_num grpcpumin_num
	global grpcpurunmin_num grpmem_num grpwall_num maxcpumin_num
	global maxwall_num maxcpuspu_num maxnodespu_num
	global part_qos mod_job_qos time_spacing selectparam
	global nthreads job_qos mod_part_qos prio_multifactor
	global tres_cpu_mult

	# We will set qostest with the qos that we will test since the some of
	# the 21.30 inc test require the qostest variable
	set qostest $part_qos

	#
	# Test GrpNode limit
	#
	set mod_job_qos(GrpNodes) 1
	set mod_part_qos(GrpNodes) $grn_num
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_1 "QOSGrpNodeLimit"
	# Reset the value to 0
	set mod_job_qos(GrpNodes) "-1"
	set mod_part_qos(GrpNodes) "-1"

	#
	# Test GrpCpus
	#
	set mod_job_qos(GrpCpus) 1
	set mod_part_qos(GrpCpus) $grcpu_num
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_2 "QOSGrpCpuLimit"
	set mod_job_qos(GrpCpus) "-1"
	set mod_part_qos(GrpCpus) "-1"

	#
	# test GrpJob limits
	#
	set mod_job_qos(GrpJobs) 1
	set mod_part_qos(GrpJobs) $grjobs_num
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_3 "QOSGrpJobsLimit"
	set mod_job_qos(GrpJobs) "-1"
	set mod_part_qos(GrpJobs) "-1"

	#
	# test GrpSubmit
	#
	set mod_job_qos(GrpSubmit) 1
	set mod_part_qos(GrpSubmit) $grsub_num
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_4
	set mod_job_qos(GrpSubmit) "-1"
	set mod_part_qos(GrpSubmit) "-1"

	#
	# Test MaxCpus limits
	#
	set mod_job_qos(MaxCpus) 1
	set mod_part_qos(MaxCpus) $maxcpu_num
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_5 "QOSMaxCpuPerJobLimit"
	set mod_job_qos(MaxCpus) "-1"
	set mod_part_qos(MaxCpus) "-1"

	#
	# Test MaxNode limit
	#
	set mod_job_qos(MaxNodes) 1
	set mod_part_qos(MaxNodes) $maxnode_num
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_6 [list "QOSMaxNodePerJobLimit" "PartitionConfig"]
	set mod_job_qos(MaxNodes) "-1"
	set mod_part_qos(MaxNodes) "-1"

	#
	# Test MaxJobs limit
	#
	set mod_job_qos(MaxJobs) 1
	set mod_part_qos(MaxJobs) $maxjobs_num
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_7 "QOSMaxJobsPerUserLimit"
	set mod_job_qos(MaxJobs) "-1"
	set mod_part_qos(MaxJobs) "-1"

	#
	# Test MaxJobsSubmits limit
	#
	set mod_job_qos(MaxSubmitJobs) 1
	set mod_part_qos(MaxSubmitJobs) $maxjobsub_num
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_8
	set mod_job_qos(MaxSubmitJobs) "-1"
	set mod_part_qos(MaxSubmitJobs) "-1"

	#
	# Test GroupCPUMins
	#
	set mod_job_qos(GrpCpuMin) 1
	set mod_part_qos(GrpCpuMin) $grpcpumin_num
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_34_1 $part_qos "QOSGrpCPUMinutesLimit"
	set mod_job_qos(GrpCpuMin) "-1"
	set mod_part_qos(GrpCpuMin) "-1"

	#
	# Test GroupCPURunMins
	# Requires priority/multifactor to properly handle decay well
	#
	if { $prio_multifactor != 0 } {
		set mod_job_qos(GrpCpuRunMin) 1
		set mod_part_qos(GrpCpuRunMin) $grpcpurunmin_num
		mod_qos $part_qos [array get mod_part_qos]
		mod_qos $job_qos [array get mod_job_qos]
		sleep $time_spacing
		inc21_30_10 "QOSGrpCPURunMinutesLimit"
		set mod_job_qos(GrpCpuRunMin) "-1"
		set mod_part_qos(GrpCpuRunMin) "-1"
	}

	#
	# Test Group Memory
	#
	set mod_job_qos(GrpMem) 1
	set mod_part_qos(GrpMem) $grpmem_num
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_11 "QOSGrpMemLimit"
	set mod_job_qos(GrpMem) "-1"
	set mod_part_qos(GrpMem) "-1"

	#
	# Test Group wall
	# Requires priority/multifactor to properly handle decay well
	#
	if { $prio_multifactor != 0 } {
		set mod_job_qos(GrpWall) 1
		set mod_part_qos(GrpWall) $grpwall_num
		mod_qos $part_qos [array get mod_part_qos]
		mod_qos $job_qos [array get mod_job_qos]
		sleep $time_spacing
		inc21_34_2 $part_qos
		set mod_job_qos(GrpWall) "-1"
		set mod_part_qos(GrpWall) "-1"
	}

	#
	# Test Max Cpu Mins
	#
	set mod_job_qos(MaxCpuMin) 1
	set mod_part_qos(MaxCpuMin) $maxcpumin_num
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_13 "QOSMaxCpuMinutesPerJobLimit"
	set mod_job_qos(MaxCpuMin) "-1"
	set mod_part_qos(MaxCpuMin) "-1"

	#
	# Test Max Wall
	#
	set mod_job_qos(MaxWall) 1
	set mod_part_qos(MaxWall) $maxwall_num
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_14 "QOSMaxWallDurationPerJobLimit"
	set mod_job_qos(MaxWall) "-1"
	set mod_part_qos(MaxWall) "-1"

	#
	# Test Max CPUs Per User
	#

	# If CR_CORE set maxcpuspu a multiple number of threads
	if {$selectparam} {
		set maxcpuspu_num [expr $maxcpuspu_num * $nthreads]
	}
	set mod_job_qos(MaxCpusPerUser) 1
	set mod_part_qos(MaxCpusPerUser) $maxcpuspu_num
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_15 "QOSMaxCpuPerUserLimit"
	set mod_job_qos(MaxCpusPerUser) "-1"
	set mod_part_qos(MaxCpusPerUser) "-1"

	#
	# Test MaxNodesPerUser
	#
	set mod_job_qos(MaxNodesPerUser) 1
	set mod_part_qos(MaxNodesPerUser) $maxnodespu_num
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_16 [list "QOSMaxNodePerUserLimit" "PartitionConfig"]
	set mod_job_qos(MaxNodesPerUser) "-1"
	set mod_part_qos(MaxNodesPerUser) "-1"

	#
	# QOS/Parent QOS MaxWall
	#
	set mod_job_qos(MaxWall) 1
	set mod_part_qos(MaxWall) $maxwall_num
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_17
	set mod_job_qos(MaxWall) "-1"
	set mod_part_qos(MaxWall) "-1"


	# TRESBillingWeight TESTS

	#
	# Test GrpTRES=billing
	#
	set mod_job_qos(GrpTRES=billing) 1
	set mod_part_qos(GrpTRES=billing) [expr $grcpu_num * $tres_cpu_mult]
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_2 "QOSGrpBilling"
	set mod_job_qos(GrpTRES=billing) "-1"
	set mod_part_qos(GrpTRES=billing) "-1"


	#
	# Test GrpTRESMins=billing
	#
	set mod_job_qos(GrpTRESMins=billing) 1
	set mod_part_qos(GrpTRESMins=billing) [expr $grpcpumin_num * $tres_cpu_mult]
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_34_1 $part_qos "QOSGrpBillingMinutes"
	set mod_job_qos(GrpTRESMins=billing) "-1"
	set mod_part_qos(GrpTRESMins=billing) "-1"


	#
	# Test GrpTRESRunMins=billing
	# Requires priority/multifactor to properly handle decay well
	#
	if { $prio_multifactor != 0 } {
		set mod_job_qos(GrpTRESRunMins=billing) 1
		set mod_part_qos(GrpTRESRunMins=billing) [expr $grpcpurunmin_num * $tres_cpu_mult]
		mod_qos $part_qos [array get mod_part_qos]
		mod_qos $job_qos [array get mod_job_qos]
		sleep $time_spacing
		inc21_30_10 "QOSGrpBillingRunMinutes"
		set mod_job_qos(GrpTRESRunMins=billing) "-1"
		set mod_part_qos(GrpTRESRunMins=billing) "-1"
	}


	#
	# Test MaxTRESPerJob=billing
	#
	set mod_job_qos(MaxTRESPerJob=billing) 1
	set mod_part_qos(MaxTRESPerJob=billing) [expr $maxcpu_num * $tres_cpu_mult]
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_5 "QOSMaxBillingPerJob"
	set mod_job_qos(MaxTRESPerJob=billing) "-1"
	set mod_part_qos(MaxTRESPerJob=billing) "-1"

	#
	# Test MaxTRESMinsPerJob=billing
	#
	set mod_job_qos(MaxTRESMinsPerJob=billing) 1
	set mod_part_qos(MaxTRESMinsPerJob=billing) [expr $maxcpumin_num * $tres_cpu_mult]
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_13 "QOSMaxBillingMinutesPerJob"
	set mod_job_qos(MaxTRESMinsPerJob=billing) "-1"
	set mod_part_qos(MaxTRESMinsPerJob=billing) "-1"

	#
	# Test MaxTRESPerUser=billing
	#
	# If CR_CORE set billing a multiple number of threads
	if {$selectparam} {
		set maxcpuspu_num [expr $maxcpuspu_num * $nthreads]
	}
	set mod_job_qos(MaxTRESPerUser=billing) 1
	set mod_part_qos(MaxTRESPerUser=billing) [expr $maxcpuspu_num * $tres_cpu_mult]
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_15 "QOSMaxBillingPerUser"
	set mod_job_qos(MaxTRESPerUser=billing) "-1"
	set mod_part_qos(MaxTRESPerUser=billing) "-1"
}

##########################################################
#
# Test that jobs qos limits are enforced after
# PartitionQos flag is set on the job's qos
#
##########################################################

proc qos_test { } {

	global grn_num grcpu_num grjobs_num grsub_num grsub_num maxcpu_num
	global maxnode_num maxjobs_num maxjobsub_num grpcpumin_num
	global grpcpurunmin_num grpmem_num grpwall_num maxcpumin_num
	global maxwall_num maxcpuspu_num maxnodespu_num
	global job_qos mod_job_qos time_spacing selectparam
	global nthreads part_qos mod_part_qos prio_multifactor
	global tres_cpu_mult

	# We will set qostest with the qos that we will test since the some of
	# the 21.30 inc test require the qostest variable
	set qostest $job_qos

	#
	# Test GrpNode limit
	#
	set mod_job_qos(GrpNodes) $grn_num
	set mod_part_qos(GrpNodes) 1
	mod_qos $job_qos [array get mod_job_qos]
	mod_qos $part_qos [array get mod_part_qos]
	sleep $time_spacing
	inc21_30_1 "QOSGrpNodeLimit"
	# Reset the value to 0
	set mod_job_qos(GrpNodes) "-1"
	set mod_part_qos(GrpNodes) "-1"

	#
	# Test GrpCpus
	#
	set mod_job_qos(GrpCpus) $grcpu_num
	set mod_part_qos(GrpCpus) $grcpu_num
	mod_qos $job_qos [array get mod_job_qos]
	mod_qos $part_qos [array get mod_part_qos]
	sleep $time_spacing
	inc21_30_2 "QOSGrpCpuLimit"
	set mod_job_qos(GrpCpus) "-1"
	set mod_part_qos(GrpCpus) "-1"

	#
	# test GrpJob limits
	#
	set mod_job_qos(GrpJobs) $grjobs_num
	set mod_part_qos(GrpJobs) 1
	mod_qos $job_qos [array get mod_job_qos]
	mod_qos $part_qos [array get mod_part_qos]
	sleep $time_spacing
	inc21_30_3 "QOSGrpJobsLimit"
	set mod_job_qos(GrpJobs) "-1"
	set mod_part_qos(GrpJobs) "-1"

	#
	# test GrpSubmit
	#
	set mod_job_qos(GrpSubmit) $grsub_num
	set mod_part_qos(GrpSubmit) 1
	mod_qos $job_qos [array get mod_job_qos]
	mod_qos $part_qos [array get mod_part_qos]
	sleep $time_spacing
	inc21_30_4
	set mod_job_qos(GrpSubmit) "-1"
	set mod_part_qos(GrpSubmit) "-1"

	#
	# Test MaxCpus limits
	#
	set mod_job_qos(MaxCpus) $maxcpu_num
	set mod_part_qos(MaxCpus) 1
	mod_qos $job_qos [array get mod_job_qos]
	mod_qos $part_qos [array get mod_part_qos]
	sleep $time_spacing
	inc21_30_5 "QOSMaxCpuPerJobLimit"
	set mod_job_qos(MaxCpus) "-1"
	set mod_part_qos(MaxCpus) "-1"

	#
	# Test MaxNode limit
	#
	set mod_job_qos(MaxNodes) $maxnode_num
	set mod_part_qos(MaxNodes) 1
	mod_qos $job_qos [array get mod_job_qos]
	mod_qos $part_qos [array get mod_part_qos]
	sleep $time_spacing
	inc21_30_6 [list "QOSMaxNodePerJobLimit" "PartitionConfig"]
	set mod_job_qos(MaxNodes) "-1"
	set mod_part_qos(MaxNodes) "-1"

	#
	# Test MaxJobs limit
	#
	set mod_job_qos(MaxJobs) $maxjobs_num
	set mod_part_qos(MaxJobs) 1
	mod_qos $job_qos [array get mod_job_qos]
	mod_qos $part_qos [array get mod_part_qos]
	sleep $time_spacing
	inc21_30_7 "QOSMaxJobsPerUserLimit"
	set mod_job_qos(MaxJobs) "-1"
	set mod_part_qos(MaxJobs) "-1"

	#
	# Test MaxJobsSubmits limit
	#
	set mod_job_qos(MaxSubmitJobs) $maxjobsub_num
	set mod_part_qos(MaxSubmitJobs) 1
	mod_qos $job_qos [array get mod_job_qos]
	mod_qos $part_qos [array get mod_part_qos]
	sleep $time_spacing
	inc21_30_8
	set mod_job_qos(MaxSubmitJobs) "-1"
	set mod_part_qos(MaxSubmitJobs) "-1"

	#
	# Test GroupCPUMins
	#
	set mod_job_qos(GrpCpuMin) $grpcpumin_num
	set mod_part_qos(GrpCpuMin) 1
	mod_qos $job_qos [array get mod_job_qos]
	mod_qos $part_qos [array get mod_part_qos]
	sleep $time_spacing
	inc21_34_1 $job_qos "QOSGrpCPUMinutesLimit"
	set mod_job_qos(GrpCpuMin) "-1"
	set mod_part_qos(GrpCpuMin) "-1"

	#
	# Test GroupCPURunMins
	# Requires priority/multifactor to properly handle decay well
	#
	if { $prio_multifactor != 0 } {
		set mod_job_qos(GrpCpuRunMin) $grpcpurunmin_num
		set mod_part_qos(GrpCpuRunMin) 1
		mod_qos $job_qos [array get mod_job_qos]
		mod_qos $part_qos [array get mod_part_qos]
		sleep $time_spacing
		inc21_30_10  "QOSGrpCPURunMinutesLimit"
		set mod_job_qos(GrpCpuRunMin) "-1"
		set mod_part_qos(GrpCpuRunMin) "-1"
	}

	#
	# Test Group Memory
	#
	set mod_job_qos(GrpMem) $grpmem_num
	set mod_part_qos(GrpMem) 1
	mod_qos $job_qos [array get mod_job_qos]
	mod_qos $part_qos [array get mod_part_qos]
	sleep $time_spacing
	inc21_30_11 "QOSGrpMemLimit"
	set mod_job_qos(GrpMem) "-1"
	set mod_part_qos(GrpMem) "-1"

	#
	# Test Group wall
	# Requires priority/multifactor to properly handle decay well
	#
	if { $prio_multifactor != 0 } {
		set mod_job_qos(GrpWall) $grpwall_num
		set mod_part_qos(GrpWall) 1
		mod_qos $job_qos [array get mod_job_qos]
		mod_qos $part_qos [array get mod_part_qos]
		sleep $time_spacing
		inc21_34_2 $job_qos
		set mod_job_qos(GrpWall) "-1"
		set mod_part_qos(GrpWall) "-1"
	}

	#
	# Test Max Cpu Mins
	#
	set mod_job_qos(MaxCpuMin) $maxcpumin_num
	set mod_part_qos(MaxCpuMin) 1
	mod_qos $job_qos [array get mod_job_qos]
	mod_qos $part_qos [array get mod_part_qos]
	sleep $time_spacing
	inc21_30_13 "QOSMaxCpuMinutesPerJobLimit"
	set mod_job_qos(MaxCpuMin) "-1"
	set mod_part_qos(MaxCpuMin) "-1"

	#
	# Test Max Wall
	#
	set mod_job_qos(MaxWall) $maxwall_num
	set mod_part_qos(MaxWall) 1
	mod_qos $job_qos [array get mod_job_qos]
	mod_qos $part_qos [array get mod_part_qos]
	sleep $time_spacing
	inc21_30_14 "QOSMaxWallDurationPerJobLimit"
	set mod_job_qos(MaxWall) "-1"
	set mod_part_qos(MaxWall) "-1"

	#
	# Test Max CPUs Per User
	#

	# If CR_CORE set maxcpuspu a multiple number of threads
	if {$selectparam} {
		set maxcpuspu_num [expr $maxcpuspu_num * $nthreads]
	}

	set mod_job_qos(MaxCpusPerUser) $maxcpuspu_num
	set mod_part_qos(MaxCpusPerUser) 1
	mod_qos $job_qos [array get mod_job_qos]
	mod_qos $part_qos [array get mod_part_qos]
	sleep $time_spacing
	inc21_30_15 "QOSMaxCpuPerUserLimit"
	set mod_job_qos(MaxCpusPerUser) "-1"
	set mod_part_qos(MaxCpusPerUser) "-1"

	#
	# Test MaxNodesPerUser
	#
	set mod_job_qos(MaxNodesPerUser) $maxnodespu_num
	set mod_part_qos(MaxNodesPerUser) 1
	mod_qos $job_qos [array get mod_job_qos]
	mod_qos $part_qos [array get mod_part_qos]
	sleep $time_spacing
	inc21_30_16 [list "QOSMaxNodePerUserLimit" "PartitionConfig"]
	set mod_job_qos(MaxNodesPerUser) "-1"
	set mod_part_qos(MaxNodesPerUser) "-1"

	#
	# QOS/Parent QOS MaxWall
	#
	set mod_job_qos(MaxWall) $maxwall_num
	set mod_part_qos(MaxWall) 1
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_17
	set mod_job_qos(MaxWall) "-1"
	set mod_part_qos(MaxWall) "-1"


	# TRESBillingWeight TESTS

	#
	# Test GrpTRES=billing
	#
	set mod_job_qos(GrpTRES=billing) [expr $grcpu_num * $tres_cpu_mult]
	set mod_part_qos(GrpTRES=billing) [expr $grcpu_num * $tres_cpu_mult]
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_2 "QOSGrpBilling"
	set mod_job_qos(GrpTRES=billing) "-1"
	set mod_part_qos(GrpTRES=billing) "-1"


	#
	# Test GrpTRESMins=billing
	#
	set mod_job_qos(GrpTRESMins=billing) [expr $grpcpumin_num * $tres_cpu_mult]
	set mod_part_qos(GrpTRESMins=billing) 1
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_34_1 $job_qos "QOSGrpBillingMinutes"
	set mod_job_qos(GrpTRESMins=billing) "-1"
	set mod_part_qos(GrpTRESMins=billing) "-1"


	#
	# Test GrpTRESRunMins=billing
	# Requires priority/multifactor to properly handle decay well
	#
	if { $prio_multifactor != 0 } {
		set mod_job_qos(GrpTRESRunMins=billing) [expr $grpcpurunmin_num * $tres_cpu_mult]
		set mod_part_qos(GrpTRESRunMins=billing) 1
		mod_qos $part_qos [array get mod_part_qos]
		mod_qos $job_qos [array get mod_job_qos]
		sleep $time_spacing
		inc21_30_10  "QOSGrpBillingRunMinutes"
		set mod_job_qos(GrpTRESRunMins=billing) "-1"
		set mod_part_qos(GrpTRESRunMins=billing) "-1"
	}


	#
	# Test MaxTRESPerJob=billing
	#
	set mod_job_qos(MaxTRESPerJob=billing) [expr $maxcpu_num * $tres_cpu_mult]
	set mod_part_qos(MaxTRESPerJob=billing) 1
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_5 "QOSMaxBillingPerJob"
	set mod_job_qos(MaxTRESPerJob=billing) "-1"
	set mod_part_qos(MaxTRESPerJob=billing) "-1"

	#
	# Test MaxTRESMinsPerJob=billing
	#
	set mod_job_qos(MaxTRESMinsPerJob=billing) [expr $maxcpumin_num * $tres_cpu_mult]
	set mod_part_qos(MaxTRESMinsPerJob=billing) 1
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_13 "QOSMaxBillingMinutesPerJob"
	set mod_job_qos(MaxTRESMinsPerJob=billing) "-1"
	set mod_part_qos(MaxTRESMinsPerJob=billing) "-1"

	#
	# Test MaxTRESPerUser=billing
	#
	# If CR_CORE set billing a multiple number of threads
	if {$selectparam} {
		set maxcpuspu_num [expr $maxcpuspu_num * $nthreads]
	}
	set mod_job_qos(MaxTRESPerUser=billing) [expr $maxcpuspu_num * $tres_cpu_mult]
	set mod_part_qos(MaxTRESPerUser=billing) 1
	mod_qos $part_qos [array get mod_part_qos]
	mod_qos $job_qos [array get mod_job_qos]
	sleep $time_spacing
	inc21_30_15 "QOSMaxBillingPerUser"
	set mod_job_qos(MaxTRESPerUser=billing) "-1"
	set mod_part_qos(MaxTRESPerUser=billing) "-1"
}