| /*****************************************************************************\ | |
| * gres_stepmgr.h - Functions for gres used only in the slurmctld | |
| ***************************************************************************** | |
| * Copyright (C) SchedMD LLC. | |
| * Derived in large part from code previously in interfaces/gres.h | |
| * | |
| * This file is part of Slurm, a resource management program. | |
| * For details, see <https://slurm.schedmd.com/>. | |
| * Please also read the included file: DISCLAIMER. | |
| * | |
| * Slurm is free software; you can redistribute it and/or modify it under | |
| * the terms of the GNU General Public License as published by the Free | |
| * Software Foundation; either version 2 of the License, or (at your option) | |
| * any later version. | |
| * | |
| * In addition, as a special exception, the copyright holders give permission | |
| * to link the code of portions of this program with the OpenSSL library under | |
| * certain conditions as described in each individual source file, and | |
| * distribute linked combinations including the two. You must obey the GNU | |
| * General Public License in all respects for all of the code used other than | |
| * OpenSSL. If you modify file(s) with this exception, you may extend this | |
| * exception to your version of the file(s), but you are not obligated to do | |
| * so. If you do not wish to do so, delete this exception statement from your | |
| * version. If you delete this exception statement from all source files in | |
| * the program, then also delete it here. | |
| * | |
| * Slurm is distributed in the hope that it will be useful, but WITHOUT ANY | |
| * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | |
| * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more | |
| * details. | |
| * | |
| * You should have received a copy of the GNU General Public License along | |
| * with Slurm; if not, write to the Free Software Foundation, Inc., | |
| * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | |
| \*****************************************************************************/ | |
| #ifndef _GRES_CTLD_H | |
| #define _GRES_CTLD_H | |
| #include "src/interfaces/gres.h" | |
| typedef struct gres_stepmgr_step_test_args { | |
| uint16_t cpus_per_task; /* IN cpus_per_task - number of CPUs required | |
| * per task */ | |
| int *err_code; /* OUT err_code - If an error occurred, set this to tell | |
| * the caller why the error happened. */ | |
| bool first_step_node; /* IN first_step_node - true if this is node zero | |
| * of the step (do initing) */ | |
| bool ignore_alloc; /* IN ignore_alloc - if set ignore resources already | |
| * allocated to running steps */ | |
| list_t *job_gres_list; /* IN job_gres_list - a running job's allocated gres | |
| * info */ | |
| uint32_t job_id; /* Job ID of the step being allocated */ | |
| job_resources_t *job_resrcs_ptr; /* IN job_resrcs_ptr - pointer to this | |
| * job's job_resources_t; used to know | |
| * how much of the job's memory is | |
| * available. */ | |
| int max_rem_nodes; /* IN max_rem_nodes - maximum nodes remaining for | |
| * step (including this one) */ | |
| int node_offset; /* IN node_offset - index into the job's node | |
| * allocation */ | |
| list_t *step_gres_list; /* IN/OUT step_gres_list - a pending job step's | |
| * gres requirements */ | |
| uint32_t step_id; /* ID of the step being allocated */ | |
| bool test_mem; /* IN test_mem - true if we should test if mem_per_gres | |
| * would exceed a limit. */ | |
| } gres_stepmgr_step_test_args_t; | |
| /* | |
| * Fill in job_gres_list with the total amount of GRES on a node. | |
| * OUT job_gres_list - This list will be destroyed and remade with all GRES on | |
| * node. | |
| * IN node_gres_list - node's gres_list built by | |
| * gres_node_config_validate() | |
| * IN job_id - job's ID (for logging) | |
| * IN node_name - name of the node (for logging) | |
| * RET SLURM_SUCCESS or error code | |
| */ | |
| extern int gres_stepmgr_job_select_whole_node( | |
| list_t **job_gres_list, list_t *node_gres_list, | |
| uint32_t job_id, char *node_name); | |
| /* | |
| * Select and allocate all GRES on a node to a job and update node and job GRES | |
| * information | |
| * IN job_gres_list - job's gres_list built by gres_job_whole_node(). | |
| * OUT job_alloc_gres_list - job's list of allocated gres | |
| * IN node_gres_list - node's gres_list built by | |
| * gres_node_config_validate() | |
| * IN node_cnt - total number of nodes originally allocated to the job | |
| * IN node_index - zero-origin global node index | |
| * IN node_offset - zero-origin index in job allocation to the node of interest | |
| * IN job_id - job's ID (for logging) | |
| * IN node_name - name of the node (for logging) | |
| * IN core_bitmap - cores allocated to this job on this node (NULL if not | |
| * available) | |
| * IN new_alloc - If this is a new allocation or not. | |
| * RET SLURM_SUCCESS or error code | |
| */ | |
| extern int gres_stepmgr_job_alloc_whole_node( | |
| list_t *job_gres_list, list_t **job_alloc_gres_list, | |
| list_t *node_gres_list, int node_cnt, int node_index, int node_offset, | |
| uint32_t job_id, char *node_name, | |
| bitstr_t *core_bitmap, bool new_alloc); | |
| /* | |
| * Select and allocate GRES to a job and update node and job GRES information | |
| * IN job_gres_list - job's gres_list built by gres_job_state_validate() | |
| * OUT job_alloc_gres_list - job's list of allocated gres | |
| * IN node_gres_list - node's gres_list built by | |
| * gres_node_config_validate() | |
| * IN node_cnt - total number of nodes originally allocated to the job | |
| * IN node_index - zero-origin global node index | |
| * IN node_offset - zero-origin index in job allocation to the node of interest | |
| * IN job_id - job's ID (for logging) | |
| * IN node_name - name of the node (for logging) | |
| * IN core_bitmap - cores allocated to this job on this node (NULL if not | |
| * available) | |
| * IN new_alloc - If this is a new allocation or not. | |
| * RET SLURM_SUCCESS or error code | |
| */ | |
| extern int gres_stepmgr_job_alloc( | |
| list_t *job_gres_list, list_t **job_alloc_gres_list, | |
| list_t *node_gres_list, int node_cnt, | |
| int node_index, int node_offset, | |
| uint32_t job_id, char *node_name, | |
| bitstr_t *core_bitmap, bool new_alloc); | |
| /* | |
| * Deallocate resource from a job and update node and job gres information | |
| * IN job_gres_list - job's gres_list built by gres_job_state_validate() | |
| * IN node_gres_list - node's gres_list built by | |
| * gres_node_config_validate() | |
| * IN node_offset - zero-origin index to the node of interest | |
| * IN job_id - job's ID (for logging) | |
| * IN node_name - name of the node (for logging) | |
| * IN old_job - true if job started before last slurmctld reboot. | |
| * Immediately after slurmctld restart and before the node's | |
| * registration, the GRES type and topology. This results in | |
| * some incorrect internal bookkeeping, but does not cause | |
| * failures in terms of allocating GRES to jobs. | |
| * IN user_id - job's user ID | |
| * IN: resize - True if dealloc is due to a node being removed via a job | |
| * resize; false if dealloc is due to a job test or a real job | |
| * that is terminating. | |
| * RET SLURM_SUCCESS or error code | |
| */ | |
| extern int gres_stepmgr_job_dealloc( | |
| list_t *job_gres_list, list_t *node_gres_list, | |
| int node_offset, uint32_t job_id, | |
| char *node_name, bool old_job, bool resize); | |
| /* | |
| * Merge one job's gres allocation into another job's gres allocation. | |
| * IN from_job_gres_list - List of gres records for the job being merged | |
| * into another job | |
| * IN from_job_node_bitmap - bitmap of nodes for the job being merged into | |
| * another job | |
| * IN/OUT to_job_gres_list - List of gres records for the job being merged | |
| * into job | |
| * IN to_job_node_bitmap - bitmap of nodes for the job being merged into | |
| */ | |
| extern void gres_stepmgr_job_merge( | |
| list_t *from_job_gres_list, | |
| bitstr_t *from_job_node_bitmap, | |
| list_t *to_job_gres_list, | |
| bitstr_t *to_job_node_bitmap); | |
| /* | |
| * Clear any vestigial alloc job gres state. This may be needed on job requeue. | |
| * This only clears out the allocated portions of the gres list, it does not | |
| * remove the actual items from the list. | |
| */ | |
| extern void gres_stepmgr_job_clear_alloc(list_t *job_gres_list); | |
| /* | |
| * Test whether a job's GRES allocation on a node is a deferred bind: an entry | |
| * that is count-only (gres_cnt_node_alloc set without per-device | |
| * gres_bit_alloc) for a GRES the node can bind to devices (its gres_bit_alloc | |
| * exists). A count-only entry for a GRES with no File topology is count-only | |
| * by design and does not count. | |
| * IN job_ptr - job to test | |
| * IN node_ptr - node the job's GRES is allocated on | |
| * RET true if at least one entry on this node needs a bind | |
| */ | |
| extern bool gres_stepmgr_job_needs_bind(job_record_t *job_ptr, | |
| node_record_t *node_ptr); | |
| /* | |
| * Complete a job's deferred (count-only) GRES allocation on one node by | |
| * re-running the normal allocator (dealloc + gres_stepmgr_job_alloc with | |
| * new_alloc=true) after the node registers its device topology. Handles | |
| * untyped, typed and shared GRES via the existing _job_alloc() path. | |
| * IN job_ptr - job to bind | |
| * IN node_ptr - node that registered its device topology | |
| * RET SLURM_SUCCESS if the node's GRES is now bound; SLURM_ERROR if it could | |
| * not be bound, either because the node registered no usable device | |
| * topology or because the re-allocation itself failed. | |
| */ | |
| extern int gres_stepmgr_job_realloc_node(job_record_t *job_ptr, | |
| node_record_t *node_ptr); | |
| /* Given a job's GRES data structure, return the indices for selected elements | |
| * IN job_gres_list - job's allocated GRES data structure | |
| * IN nodes - list of nodes allocated to job | |
| * OUT gres_detail_cnt - Number of elements (nodes) in gres_detail_str | |
| * OUT gres_detail_str - Description of GRES on each node | |
| * OUT total_gres_str - String containing all gres in the job and counts. | |
| */ | |
| extern void gres_stepmgr_job_build_details( | |
| list_t *job_gres_list, char *nodes, | |
| uint32_t *gres_detail_cnt, | |
| char ***gres_detail_str, | |
| char **total_gres_str); | |
| /* | |
| * Fill in the job allocated tres_cnt based off the gres_list and node_cnt | |
| * IN gres_list - filled in with gres_job_state_t's | |
| * IN node_cnt - number of nodes in the job | |
| * OUT tres_cnt - gres spots filled in with total number of TRES | |
| * requested for job that are requested in gres_list | |
| * IN locked - if the assoc_mgr tres read locked is locked or not | |
| */ | |
| extern void gres_stepmgr_set_job_tres_cnt( | |
| list_t *gres_list, uint32_t node_cnt, uint64_t *tres_cnt, bool locked); | |
| /* | |
| * Fill in the node allocated tres_cnt based off the gres_list | |
| * IN gres_list - filled in with gres_node_state_t's gres_alloc_cnt | |
| * OUT tres_cnt - gres spots filled in with total number of TRES | |
| * allocated on node | |
| * IN locked - if the assoc_mgr tres read locked is locked or not | |
| */ | |
| extern void gres_stepmgr_set_node_tres_cnt( | |
| list_t *gres_list, uint64_t *tres_cnt, bool locked); | |
| /* | |
| * Determine how many cores of a job's allocation can be allocated to a step | |
| * on a specific node | |
| * IN args - see the definition of gres_stepmgr_step_test_args_t | |
| * RET Count of available cpus on this node (sort of): | |
| * NO_VAL64 if no limit or 0 if node is not usable | |
| */ | |
| extern uint64_t gres_stepmgr_step_test(gres_stepmgr_step_test_args_t *args); | |
| /* | |
| * If a step gres request used gres_per_step it must be tested more than just in | |
| * gres_stepmgr_step_test. This function only acts when gres_per_step is used | |
| * IN step_gres_list - step's requested GRES data structure | |
| * IN job_ptr - Job data | |
| * IN/OUT nodes_avail - Bitstring of nodes available for this step to use | |
| * IN min_nodes - minimum nodes required for this step | |
| */ | |
| extern void gres_stepmgr_step_test_per_step( | |
| list_t *step_gres_list, | |
| job_record_t *job_ptr, | |
| bitstr_t *nodes_avail, | |
| int min_nodes); | |
| /* | |
| * Allocate resource to a step and update job and step gres information | |
| * IN step_gres_list - step's gres_list built by | |
| * gres_step_state_validate() | |
| * OUT step_gres_list_alloc - step's list of allocated gres | |
| * IN job_gres_list - job's allocated gres_list built by gres_stepmgr_job_alloc() | |
| * IN node_offset - job's zero-origin index to the node of interest | |
| * IN first_step_node - true if this is node zero of the step | |
| * (do initialization) | |
| * IN tasks_on_node - number of tasks to be launched on this node | |
| * IN rem_nodes - desired additional node count to allocate, including this node | |
| * IN job_ptr, step_id - ID of the step being allocated. | |
| * IN decr_job_alloc - whether or not to decrement the step allocation from the | |
| * job allocation. | |
| * OUT step_node_mem_alloc - the amount of memory allocated to the step on this | |
| * node based on mem_per_gres requirements. | |
| * IN node_gres_list - node's gres list | |
| * IN core_bitmap - bitmap of all cores available for the step | |
| * OUT total_gres_cpu_cnt - sum of (cpus_per_gres * alloc_gres_cnt) for each | |
| * gres allocated to the step on this node | |
| * RET SLURM_SUCCESS or error code | |
| */ | |
| extern int gres_stepmgr_step_alloc(list_t *step_gres_list, | |
| list_t **step_gres_list_alloc, | |
| list_t *job_gres_list, int node_offset, | |
| bool first_step_node, uint16_t tasks_on_node, | |
| uint32_t rem_nodes, job_record_t *job_ptr, | |
| uint32_t step_id, bool decr_job_alloc, | |
| uint64_t *step_node_mem_alloc, | |
| list_t *node_gres_list, | |
| bitstr_t *core_bitmap, | |
| int *total_gres_cpu_cnt); | |
| /* | |
| * Deallocate resource to a step and update job and step gres information | |
| * IN step_gres_list_alloc - step's list of allocated gres | |
| * IN job_gres_list - job's allocated gres_list built by | |
| * gres_stepmgr_job_alloc() | |
| * IN job_id, step_id - ID of the step being allocated. | |
| * IN node_offset - job's zero-origin index to the node of interest | |
| * IN decr_job_alloc - whether or not to decrement the step allocation from the | |
| * job allocation. | |
| * RET SLURM_SUCCESS or error code | |
| */ | |
| extern int gres_stepmgr_step_dealloc(list_t *step_gres_list_alloc, | |
| list_t *job_gres_list, | |
| uint32_t job_id, uint32_t step_id, | |
| int node_offset, bool decr_job_alloc); | |
| /* | |
| * A job allocation size has changed. Update the job step gres information | |
| * bitmaps and other data structures. | |
| * IN gres_list - List of Gres records for this step to track usage | |
| * IN orig_job_node_bitmap - bitmap of nodes in the original job allocation | |
| * IN new_job_node_bitmap - bitmap of nodes in the new job allocation | |
| */ | |
| extern void gres_stepmgr_step_state_rebase( | |
| list_t *gres_list, | |
| bitstr_t *orig_job_node_bitmap, | |
| bitstr_t *new_job_node_bitmap); | |
| /* | |
| * Given a job's GRES data structure, return a simple tres string of gres | |
| * allocated on the node_inx requested | |
| * IN job_gres_list - job's allocated GRES data structure | |
| * IN node_inx - position of node in job_state_ptr->gres_cnt_node_alloc | |
| * | |
| * RET - simple string containing gres this job is allocated on the node | |
| * requested. | |
| */ | |
| extern char *gres_stepmgr_gres_on_node_as_tres( | |
| list_t *job_gres_list, int node_inx, bool locked); | |
| /* | |
| * Translate a gres_list into a tres_str | |
| * IN gres_list - filled in with gres_job_state_t or gres_step_state_t's | |
| * IN locked - if the assoc_mgr tres read locked is locked or not | |
| * RET char * in a simple TRES format | |
| */ | |
| extern char *gres_stepmgr_gres_2_tres_str( | |
| list_t *gres_list, bool locked); | |
| #endif /* _GRES_CTLD_H */ |