#!/usr/bin/expect
############################################################################
# Purpose: Test of Slurm functionality
#          Confirm that FLEX flag of reservations works correctly
############################################################################
# Copyright (C) 2002 The Regents of the University of California.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# CODE-OCEC-09-009. All rights reserved.
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set job_id               0
set user_name            [get_my_user_name]
set default_partition    [default_partition]
set res_name             "res_$test_name"
set resv_over_run        false
set config_file          [get_conf_path]/slurm.conf

if { [get_config_param ResvOverRun] != "0 min" } {
	set resv_over_run true
}

proc cleanup {} {
	global resv_over_run config_file

	subcleanup

	if { $resv_over_run } {
		restore_conf $config_file
		reconfigure -fail
	}
}

proc subcleanup {} {
	global scontrol res_name job_id

	cancel_job $job_id
	run_command "$scontrol delete reservation=$res_name"
}

proc setup {use_flex {start_time ""}} {
	global res_name user_name default_partition

	if {$use_flex} {
		log_info "Creating a reservation with Flags=FLEX"
		set flags "Flags=FLEX"
	} else {
		log_info "Creating a reservation WITHOUT Flags=FLEX"
		set flags ""
	}
	if {[create_res $res_name "StartTime=now$start_time Duration=1 NodeCnt=1 partition=$default_partition user=$user_name $flags"]} {
		fail "Unable to create a reservation with flags=FLEX"
	}
}

set nb_nodes [get_partition_param [default_partition] "TotalNodes"]
if {$nb_nodes < 2} {
	skip "Need 2 or more nodes in default partition"
}

if { $resv_over_run } {
	save_conf $config_file
	exec $bin_echo ResvOverRun=0 >> $config_file
	reconfigure -fail
}

proc test_flex {use_flex submit_active} {
	global res_name job_id resv_over_run

	# Subtest job using more nodes than reservation
	subcleanup
	if {$submit_active} {
		setup $use_flex ""
		wait_for {[get_reservation_param $res_name State] eq "ACTIVE"} {}
	} else {
		setup $use_flex "+5"
	}
	set job_id [submit_job -none "-N2 --reservation=$res_name --wrap 'sleep 30' -o /dev/null -e /dev/null"]
	if {$use_flex} {
		subtest {![wait_for_job -timeout 30 $job_id RUNNING]} "Job should be running because with FLEX more nodes can be requested"
	} else {
		if {$submit_active} {
			subtest {$job_id == 0} "Job shouldn't be running requested more nodes than in reservation without FLEX and ACTIVE"
		} else {
			subtest {[wait_for_job -timeout 30 $job_id RUNNING]} "Job shouldn't be running because requested more nodes than in reservation without FLEX"
		}
	}

	# Subtest job with EndTime after reservation end
	subcleanup
	if {$submit_active} {
		setup $use_flex ""
		wait_for {[get_reservation_param $res_name State] eq "ACTIVE"} {}
	} else {
		setup $use_flex "+5"
	}
	set job_id [submit_job -fail "-N1 --reservation=$res_name -t 120 --wrap 'sleep 100' -o /dev/null -e /dev/null"]
	subtest {![wait_for_job -timeout 30 $job_id RUNNING]} "Job should be RUNNING"
	subtest {![wait_for_job -timeout 200 -pollinterval 10 $job_id DONE]} "Job should be DONE"

	if {$use_flex} {
		subtest {[get_job_param $job_id JobState] == "COMPLETED" } "Job should complete successfully after reservation end when it was FLEX"
	} else {
		subtest {[get_job_param $job_id JobState] == "TIMEOUT" } "Job should TIMEOUT with reservation end without FLEX"
	}

	# Subtest Job start before reservation StartTime
	subcleanup
	setup $use_flex "+60"
	set job_id [submit_job -fail "-N1 --reservation=$res_name -t 300 --wrap 'sleep 30' -o /dev/null -e /dev/null"]
	if {$use_flex} {
		subtest {![wait_for_job -timeout 30 $job_id RUNNING]} "Job should start before reservation since it's FLEX."
	} else {
		subtest {[wait_for_job -timeout 30 $job_id RUNNING]}  "Job shouldn't started before reservation since it's not FLEX."
	}
}

# Test with FLEX flag and when reservation is ACTIVE
test_flex true true
# Test with FLEX flag and when reservation is INACTIVE
test_flex true false
# Test without FLEX flag and when reservation is ACTIVE
test_flex false true
# Test without FLEX flag and when reservation is INACTIVE
test_flex false false