#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
#          Validates that sacctmgr show runawayjobs returns runawayjobs
############################################################################
# Copyright (C) SchedMD LLC.
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals
source ./globals_accounting

set sql_query        "$test_dir/query.sql"
set sql_query_rem    "$test_dir/query-rem.sql"
set cluster          [get_config_param "ClusterName"]
set account          "$test_name\_acct"
set user             "$test_name\_user"

# Account requirements
set acct_req(cluster) $cluster

# User requirements
set user_req(account) $account
set user_req(cluster) $cluster

if {[get_config_param "AccountingStorageType"] ne "accounting_storage/slurmdbd"} {
	skip "This test can't be run without AccountStorageType=slurmdbd"
}
if {[get_admin_level] ne "Administrator"} {
	skip "This test can't be run without being an Accounting administrator"
}

proc cleanup { } {
	global cluster account user sql_query sql_query_rem

	if {[file exists $sql_query_rem]} {
		archive_load $sql_query_rem
	}
	remove_user "" "" $user
	remove_acct "" $account
}

cleanup

# Make SQL file to remove the inserted job
set file [open $sql_query_rem "w"]
# This SQL statement is correct for our purposes (missing && in front of cluster)
puts $file "delete from job_table where alloc_nodes=\'test21.36_node\' cluster=\'$cluster\';"
close $file

# Add Test Account
if {[add_acct $account [array get acct_req]]} {
	fail "Unable to add test account ($account)"
}

# Add Test User
if {[add_user $user [array get user_req]]} {
	fail "Unable to add test user ($user)"
}

set assoc_id [get_assoc_id $user $account $cluster]

if {$assoc_id == -1} {
	skip "Unable to find $user association"
}

# DON'T MESS WITH THIS UNLESS YOU REALLY UNDERSTAND WHAT YOU ARE DOING!!!!!
# THIS COULD SERIOUSLY MESS UP YOUR DATABASE IF YOU ALTER THIS INCORRECTLY
# JUST A FRIENDLY REMINDER ;)

# SINCE THIS TEST HAS TO PUT A JOB INTO THE ACTUAL CLUSTER INSTEAD
# OF A TEST CLUSTER THIS IS MORE IMPORTANT TO GET RIGHT.

# Insert a job that has no end time and is still in running state
set file [open $sql_query "w"]
puts $file "insert into job_table (jobid, associd, cluster, uid, gid, `partition`, account, eligible, submit, start, end, name, state, priority, tres_req, tres_alloc, nodelist, alloc_nodes, qos, deleted) values"
puts $file "('65536', '$assoc_id', '$cluster', '1002', '1002', 'debug', '$account', 1199170800, 1199170800, 1199170800, 0, 'test21.32_job1', '1', '2', '1=2,2=200,4=1', '1=2,2=200,4=1', 'test21.36_node', 'test21.36_node', '0', '0')"
puts $file "on duplicate key update id=LAST_INSERT_ID(id), eligible=VALUES(eligible), submit=VALUES(submit), start=VALUES(start), end=VALUES(end), associd=VALUES(associd), tres_alloc=VALUES(tres_alloc);"
close $file

# Load a job to the database that has no end time.
if {[archive_load $sql_query]} {
	fail "Unable to load job to database"
}

# Test sacctmgr show runawayjobs
set output [run_command_output -fail "$sacctmgr show -i runawayjobs $cluster"]
subtest {[regexp "NOTE: Runaway jobs" $output]} "sacctmgr should detect runaway jobs"

# Check that jobs were fixed
set output [run_command_output -fail "$sacctmgr show -i runawayjobs $cluster"]
subtest {[regexp "No runaway jobs found" $output]} "sacctmgr should NOT detect runaway jobs"