/*****************************************************************************\
 *  gres_stepmgr.h - Functions for gres used only in the slurmctld
 *****************************************************************************
 *  Copyright (C) SchedMD LLC.
 *  Derived in large part from code previously in interfaces/gres.h
 *
 *  This file is part of Slurm, a resource management program.
 *  For details, see <https://slurm.schedmd.com/>.
 *  Please also read the included file: DISCLAIMER.
 *
 *  Slurm is free software; you can redistribute it and/or modify it under
 *  the terms of the GNU General Public License as published by the Free
 *  Software Foundation; either version 2 of the License, or (at your option)
 *  any later version.
 *
 *  In addition, as a special exception, the copyright holders give permission
 *  to link the code of portions of this program with the OpenSSL library under
 *  certain conditions as described in each individual source file, and
 *  distribute linked combinations including the two. You must obey the GNU
 *  General Public License in all respects for all of the code used other than
 *  OpenSSL. If you modify file(s) with this exception, you may extend this
 *  exception to your version of the file(s), but you are not obligated to do
 *  so. If you do not wish to do so, delete this exception statement from your
 *  version.  If you delete this exception statement from all source files in
 *  the program, then also delete it here.
 *
 *  Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
 *  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 *  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 *  details.
 *
 *  You should have received a copy of the GNU General Public License along
 *  with Slurm; if not, write to the Free Software Foundation, Inc.,
 *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
\*****************************************************************************/

#ifndef _GRES_CTLD_H
#define _GRES_CTLD_H

#include "src/interfaces/gres.h"

typedef struct gres_stepmgr_step_test_args {
	uint16_t cpus_per_task; /* IN cpus_per_task - number of CPUs required
				 * per task */
	int *err_code; /* OUT err_code - If an error occurred, set this to tell
			* the caller why the error happened. */
	bool first_step_node; /* IN first_step_node - true if this is node zero
			       * of the step (do initing) */
	bool ignore_alloc; /* IN ignore_alloc - if set ignore resources already
			    * allocated to running steps */
	list_t *job_gres_list; /* IN job_gres_list - a running job's allocated gres
				* info */
	uint32_t job_id; /* Job ID of the step being allocated */
	job_resources_t *job_resrcs_ptr; /* IN job_resrcs_ptr - pointer to this
					  * job's job_resources_t; used to know
					  * how much of the job's memory is
					  * available. */
	int max_rem_nodes; /* IN max_rem_nodes - maximum nodes remaining for
			    * step (including this one) */
	int node_offset; /* IN node_offset - index into the job's node
			  * allocation */
	list_t *step_gres_list; /* IN/OUT step_gres_list - a pending job step's
				 * gres requirements */
	uint32_t step_id; /* ID of the step being allocated */
	bool test_mem; /* IN test_mem - true if we should test if mem_per_gres
			* would exceed a limit. */
} gres_stepmgr_step_test_args_t;

/*
 * Fill in job_gres_list with the total amount of GRES on a node.
 * OUT job_gres_list - This list will be destroyed and remade with all GRES on
 *                     node.
 * IN node_gres_list - node's gres_list built by
 *		       gres_node_config_validate()
 * IN job_id      - job's ID (for logging)
 * IN node_name   - name of the node (for logging)
 * RET SLURM_SUCCESS or error code
 */
extern int gres_stepmgr_job_select_whole_node(
	list_t **job_gres_list, list_t *node_gres_list,
	uint32_t job_id, char *node_name);

/*
 * Select and allocate all GRES on a node to a job and update node and job GRES
 * information
 * IN job_gres_list - job's gres_list built by gres_job_whole_node().
 * OUT job_alloc_gres_list - job's list of allocated gres
 * IN node_gres_list - node's gres_list built by
 *		       gres_node_config_validate()
 * IN node_cnt    - total number of nodes originally allocated to the job
 * IN node_index  - zero-origin global node index
 * IN node_offset - zero-origin index in job allocation to the node of interest
 * IN job_id      - job's ID (for logging)
 * IN node_name   - name of the node (for logging)
 * IN core_bitmap - cores allocated to this job on this node (NULL if not
 *                  available)
 * IN new_alloc   - If this is a new allocation or not.
 * RET SLURM_SUCCESS or error code
 */
extern int gres_stepmgr_job_alloc_whole_node(
	list_t *job_gres_list, list_t **job_alloc_gres_list,
	list_t *node_gres_list, int node_cnt, int node_index, int node_offset,
	uint32_t job_id, char *node_name,
	bitstr_t *core_bitmap, bool new_alloc);

/*
 * Select and allocate GRES to a job and update node and job GRES information
 * IN job_gres_list - job's gres_list built by gres_job_state_validate()
 * OUT job_alloc_gres_list - job's list of allocated gres
 * IN node_gres_list - node's gres_list built by
 *		       gres_node_config_validate()
 * IN node_cnt    - total number of nodes originally allocated to the job
 * IN node_index  - zero-origin global node index
 * IN node_offset - zero-origin index in job allocation to the node of interest
 * IN job_id      - job's ID (for logging)
 * IN node_name   - name of the node (for logging)
 * IN core_bitmap - cores allocated to this job on this node (NULL if not
 *                  available)
 * IN new_alloc   - If this is a new allocation or not.
 * RET SLURM_SUCCESS or error code
 */
extern int gres_stepmgr_job_alloc(
	list_t *job_gres_list, list_t **job_alloc_gres_list,
	list_t *node_gres_list, int node_cnt,
	int node_index, int node_offset,
	uint32_t job_id, char *node_name,
	bitstr_t *core_bitmap, bool new_alloc);

/*
 * Deallocate resource from a job and update node and job gres information
 * IN job_gres_list - job's gres_list built by gres_job_state_validate()
 * IN node_gres_list - node's gres_list built by
 *		gres_node_config_validate()
 * IN node_offset - zero-origin index to the node of interest
 * IN job_id      - job's ID (for logging)
 * IN node_name   - name of the node (for logging)
 * IN old_job     - true if job started before last slurmctld reboot.
 *		    Immediately after slurmctld restart and before the node's
 *		    registration, the GRES type and topology. This results in
 *		    some incorrect internal bookkeeping, but does not cause
 *		    failures in terms of allocating GRES to jobs.
 * IN user_id     - job's user ID
 * IN: resize     - True if dealloc is due to a node being removed via a job
 * 		    resize; false if dealloc is due to a job test or a real job
 * 		    that is terminating.
 * RET SLURM_SUCCESS or error code
 */
extern int gres_stepmgr_job_dealloc(
	list_t *job_gres_list, list_t *node_gres_list,
	int node_offset, uint32_t job_id,
	char *node_name, bool old_job, bool resize);

/*
 * Merge one job's gres allocation into another job's gres allocation.
 * IN from_job_gres_list - List of gres records for the job being merged
 *			   into another job
 * IN from_job_node_bitmap - bitmap of nodes for the job being merged into
 *			     another job
 * IN/OUT to_job_gres_list - List of gres records for the job being merged
 *			     into job
 * IN to_job_node_bitmap - bitmap of nodes for the job being merged into
 */
extern void gres_stepmgr_job_merge(
	list_t *from_job_gres_list,
	bitstr_t *from_job_node_bitmap,
	list_t *to_job_gres_list,
	bitstr_t *to_job_node_bitmap);

/*
 * Clear any vestigial alloc job gres state. This may be needed on job requeue.
 * This only clears out the allocated portions of the gres list, it does not
 * remove the actual items from the list.
 */
extern void gres_stepmgr_job_clear_alloc(list_t *job_gres_list);

/* Given a job's GRES data structure, return the indices for selected elements
 * IN job_gres_list  - job's allocated GRES data structure
 * IN nodes - list of nodes allocated to job
 * OUT gres_detail_cnt - Number of elements (nodes) in gres_detail_str
 * OUT gres_detail_str - Description of GRES on each node
 * OUT total_gres_str - String containing all gres in the job and counts.
 */
extern void gres_stepmgr_job_build_details(
	list_t *job_gres_list, char *nodes,
	uint32_t *gres_detail_cnt,
	char ***gres_detail_str,
	char **total_gres_str);

/*
 * Fill in the job allocated tres_cnt based off the gres_list and node_cnt
 * IN gres_list - filled in with gres_job_state_t's
 * IN node_cnt - number of nodes in the job
 * OUT tres_cnt - gres spots filled in with total number of TRES
 *                requested for job that are requested in gres_list
 * IN locked - if the assoc_mgr tres read locked is locked or not
 */
extern void gres_stepmgr_set_job_tres_cnt(
	list_t *gres_list, uint32_t node_cnt, uint64_t *tres_cnt, bool locked);

/*
 * Fill in the node allocated tres_cnt based off the gres_list
 * IN gres_list - filled in with gres_node_state_t's gres_alloc_cnt
 * OUT tres_cnt - gres spots filled in with total number of TRES
 *                allocated on node
 * IN locked - if the assoc_mgr tres read locked is locked or not
 */
extern void gres_stepmgr_set_node_tres_cnt(
	list_t *gres_list, uint64_t *tres_cnt, bool locked);

/*
 * Determine how many cores of a job's allocation can be allocated to a step
 *	on a specific node
 * IN args - see the definition of gres_stepmgr_step_test_args_t
 * RET Count of available cpus on this node (sort of):
 *     NO_VAL64 if no limit or 0 if node is not usable
 */
extern uint64_t gres_stepmgr_step_test(gres_stepmgr_step_test_args_t *args);

/*
 * If a step gres request used gres_per_step it must be tested more than just in
 * gres_stepmgr_step_test. This function only acts when gres_per_step is used
 * IN step_gres_list  - step's requested GRES data structure
 * IN job_ptr - Job data
 * IN/OUT nodes_avail - Bitstring of nodes available for this step to use
 * IN min_nodes - minimum nodes required for this step
 */
extern void gres_stepmgr_step_test_per_step(
	list_t *step_gres_list,
	job_record_t *job_ptr,
	bitstr_t *nodes_avail,
	int min_nodes);

/*
 * Allocate resource to a step and update job and step gres information
 * IN step_gres_list - step's gres_list built by
 *		gres_step_state_validate()
 * OUT step_gres_list_alloc - step's list of allocated gres
 * IN job_gres_list - job's allocated gres_list built by gres_stepmgr_job_alloc()
 * IN node_offset - job's zero-origin index to the node of interest
 * IN first_step_node - true if this is node zero of the step
 *                      (do initialization)
 * IN tasks_on_node - number of tasks to be launched on this node
 * IN rem_nodes - desired additional node count to allocate, including this node
 * IN job_id, step_id - ID of the step being allocated.
 * IN decr_job_alloc - whether or not to decrement the step allocation from the
 *                     job allocation.
 * OUT step_node_mem_alloc - the amount of memory allocated to the step on this
 * 		node based on mem_per_gres requirements.
 * IN node_gres_list - node's gres list
 * IN core_bitmap - bitmap of all cores available for the step
 * OUT total_gres_cpu_cnt - sum of (cpus_per_gres * alloc_gres_cnt) for each
 *                          gres allocated to the step on this node
 * RET SLURM_SUCCESS or error code
 */
extern int gres_stepmgr_step_alloc(
	list_t *step_gres_list,
	list_t **step_gres_list_alloc,
	list_t *job_gres_list,
	int node_offset, bool first_step_node,
	uint16_t tasks_on_node, uint32_t rem_nodes,
	uint32_t job_id, uint32_t step_id,
	bool decr_job_alloc,
	uint64_t *step_node_mem_alloc,
	list_t *node_gres_list,
	bitstr_t *core_bitmap,
	int *total_gres_cpu_cnt);

/*
 * Deallocate resource to a step and update job and step gres information
 * IN step_gres_list_alloc - step's list of allocated gres
 * IN job_gres_list - job's allocated gres_list built by
 *                    gres_stepmgr_job_alloc()
 * IN job_id, step_id - ID of the step being allocated.
 * IN node_offset - job's zero-origin index to the node of interest
 * IN decr_job_alloc - whether or not to decrement the step allocation from the
 *                     job allocation.
 * RET SLURM_SUCCESS or error code
 */
extern int gres_stepmgr_step_dealloc(list_t *step_gres_list_alloc,
				     list_t *job_gres_list,
				     uint32_t job_id, uint32_t step_id,
				     int node_offset, bool decr_job_alloc);

/*
 * A job allocation size has changed. Update the job step gres information
 * bitmaps and other data structures.
 * IN gres_list - List of Gres records for this step to track usage
 * IN orig_job_node_bitmap - bitmap of nodes in the original job allocation
 * IN new_job_node_bitmap - bitmap of nodes in the new job allocation
 */
extern void gres_stepmgr_step_state_rebase(
	list_t *gres_list,
	bitstr_t *orig_job_node_bitmap,
	bitstr_t *new_job_node_bitmap);

/*
 * Given a job's GRES data structure, return a simple tres string of gres
 * allocated on the node_inx requested
 * IN job_gres_list  - job's allocated GRES data structure
 * IN node_inx - position of node in job_state_ptr->gres_cnt_node_alloc
 *
 * RET - simple string containing gres this job is allocated on the node
 * requested.
 */
extern char *gres_stepmgr_gres_on_node_as_tres(
	list_t *job_gres_list, int node_inx, bool locked);

/*
 * Translate a gres_list into a tres_str
 * IN gres_list - filled in with gres_job_state_t or gres_step_state_t's
 * IN locked - if the assoc_mgr tres read locked is locked or not
 * RET char * in a simple TRES format
 */
extern char *gres_stepmgr_gres_2_tres_str(
	list_t *gres_list, bool locked);

#endif /* _GRES_CTLD_H */