| /*****************************************************************************\ | |
| * gres_c_s.c - common functions for shared gres plugins | |
| ***************************************************************************** | |
| * Copyright (C) SchedMD LLC. | |
| * | |
| * This file is part of Slurm, a resource management program. | |
| * For details, see <https://slurm.schedmd.com/>. | |
| * Please also read the included file: DISCLAIMER. | |
| * | |
| * Slurm is free software; you can redistribute it and/or modify it under | |
| * the terms of the GNU General Public License as published by the Free | |
| * Software Foundation; either version 2 of the License, or (at your option) | |
| * any later version. | |
| * | |
| * In addition, as a special exception, the copyright holders give permission | |
| * to link the code of portions of this program with the OpenSSL library under | |
| * certain conditions as described in each individual source file, and | |
| * distribute linked combinations including the two. You must obey the GNU | |
| * General Public License in all respects for all of the code used other than | |
| * OpenSSL. If you modify file(s) with this exception, you may extend this | |
| * exception to your version of the file(s), but you are not obligated to do | |
| * so. If you do not wish to do so, delete this exception statement from your | |
| * version. If you delete this exception statement from all source files in | |
| * the program, then also delete it here. | |
| * | |
| * Slurm is distributed in the hope that it will be useful, but WITHOUT ANY | |
| * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | |
| * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more | |
| * details. | |
| * | |
| * You should have received a copy of the GNU General Public License along | |
| * with Slurm; if not, write to the Free Software Foundation, Inc., | |
| * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | |
| \*****************************************************************************/ | |
| #include "gres_c_s.h" | |
| list_t *shared_info = NULL; | |
| static gres_slurmd_conf_t *_create_shared_rec( | |
| gres_slurmd_conf_t *sharing_record, char *shared_name, | |
| gres_slurmd_conf_t *shared_record_in) | |
| { | |
| gres_slurmd_conf_t *shared_record = xmalloc(sizeof(gres_slurmd_conf_t)); | |
| shared_record->config_flags = sharing_record->config_flags; | |
| if (shared_record_in && | |
| gres_id_shared(shared_record_in->config_flags)) { | |
| shared_record->config_flags |= shared_record_in->config_flags; | |
| } else { | |
| shared_record->config_flags |= GRES_CONF_SHARED; | |
| /* The default for MPS is to have only one gpu sharing */ | |
| if (!xstrcasecmp(shared_name, "mps")) | |
| shared_record->config_flags |= GRES_CONF_ONE_SHARING; | |
| } | |
| shared_record->cpu_cnt = sharing_record->cpu_cnt; | |
| shared_record->cpus = xstrdup(sharing_record->cpus); | |
| if (sharing_record->cpus_bitmap) { | |
| shared_record->cpus_bitmap = | |
| bit_copy(sharing_record->cpus_bitmap); | |
| } | |
| shared_record->file = xstrdup(sharing_record->file); | |
| shared_record->name = xstrdup(shared_name); | |
| shared_record->plugin_id = gres_build_id(shared_name); | |
| shared_record->type_name = xstrdup(sharing_record->type_name); | |
| return shared_record; | |
| } | |
| /* Distribute MPS Count to records on original list */ | |
| static void _distribute_count(list_t *gres_conf_list, list_t *sharing_conf_list, | |
| uint64_t count, | |
| gres_slurmd_conf_t *shared_record_in) | |
| { | |
| gres_slurmd_conf_t *sharing_record, *shared_record; | |
| int rem_sharings = list_count(sharing_conf_list); | |
| while ((sharing_record = list_pop(sharing_conf_list))) { | |
| shared_record = _create_shared_rec(sharing_record, | |
| shared_record_in->name, | |
| shared_record_in); | |
| shared_record->count = count / rem_sharings; | |
| count -= shared_record->count; | |
| rem_sharings--; | |
| list_append(gres_conf_list, shared_record); | |
| list_append(gres_conf_list, sharing_record); | |
| } | |
| } | |
| static int _find_matching_file_gres(void *x, void *arg) | |
| { | |
| gres_slurmd_conf_t *gres_slurmd_conf1 = x; | |
| gres_slurmd_conf_t *gres_slurmd_conf2 = arg; | |
| if (!xstrcmp(gres_slurmd_conf1->file, gres_slurmd_conf2->file)) | |
| return 1; | |
| return 0; | |
| } | |
| /* | |
| * Find a shared record (mps or shard) whose Type is a substring of the given | |
| * device type, mirroring how GPU types match the detected device names. | |
| */ | |
| static int _find_shared_by_sharing_type(void *x, void *arg) | |
| { | |
| gres_slurmd_conf_t *gres_slurmd_conf = x; | |
| char *sharing_type = arg; | |
| if (gres_slurmd_conf->type_name && | |
| xstrcasestr(sharing_type, gres_slurmd_conf->type_name)) | |
| return 1; | |
| return 0; | |
| } | |
| static int _find_same_type_gres(void *x, void *arg) | |
| { | |
| gres_slurmd_conf_t *gres_slurmd_conf1 = x; | |
| gres_slurmd_conf_t *gres_slurmd_conf2 = arg; | |
| if (!xstrcasecmp(gres_slurmd_conf1->type_name, | |
| gres_slurmd_conf2->type_name)) | |
| return 1; | |
| return 0; | |
| } | |
| /* | |
| * Sort shared records (mps or shard) by Type length, longest first, | |
| * so the most specific configuration wins when several Types match one device. | |
| */ | |
| static int _sort_shared_by_type_len(void *x, void *y) | |
| { | |
| gres_slurmd_conf_t *gres_slurmd_conf1 = *(gres_slurmd_conf_t **) x; | |
| gres_slurmd_conf_t *gres_slurmd_conf2 = *(gres_slurmd_conf_t **) y; | |
| size_t len1 = gres_slurmd_conf1->type_name ? | |
| strlen(gres_slurmd_conf1->type_name) : | |
| 0; | |
| size_t len2 = gres_slurmd_conf2->type_name ? | |
| strlen(gres_slurmd_conf2->type_name) : | |
| 0; | |
| if (len1 < len2) | |
| return 1; | |
| if (len1 > len2) | |
| return -1; | |
| return 0; | |
| } | |
| /* Find a sharing (gpu) device whose type contains the shared record's Type */ | |
| static int _find_sharing_by_shared_type(void *x, void *arg) | |
| { | |
| gres_slurmd_conf_t *sharing_record = x; | |
| gres_slurmd_conf_t *shared_record = arg; | |
| if (sharing_record->type_name && | |
| xstrcasestr(sharing_record->type_name, shared_record->type_name)) | |
| return 1; | |
| return 0; | |
| } | |
| static int _foreach_warn_unmatched_type(void *x, void *arg) | |
| { | |
| gres_slurmd_conf_t *shared_record = x; | |
| list_t *sharing_conf_list = arg; | |
| if (!list_find_first(sharing_conf_list, _find_sharing_by_shared_type, | |
| shared_record)) | |
| error("Discarding gres/'shared' configuration (Type=%s) without matching gres/'sharing' device type", | |
| shared_record->type_name); | |
| return 0; | |
| } | |
| /* | |
| * Give every sharing (gpu) device the per-device Count from the configuration | |
| * matching its Type. Unlike _distribute_count(), the Count is not divided, it | |
| * applies directly to each matching device, so one configuration line covers | |
| * any number of devices of that model. Sharing devices with no matching | |
| * configuration get no shared record. | |
| * | |
| * "Name=shard Type=a100 Count=32" or "Name=mps Type=a100 Count=32" in gres.conf | |
| * will give 32 to every a100 device. | |
| */ | |
| static void _distribute_count_per_type(list_t *gres_conf_list, | |
| list_t *sharing_conf_list, | |
| list_t *shared_conf_list, | |
| char *shared_name) | |
| { | |
| gres_slurmd_conf_t *sharing_record, *shared_record, *typed_record; | |
| /* Longest Type first so the most specific configuration wins */ | |
| list_sort(shared_conf_list, _sort_shared_by_type_len); | |
| (void) list_for_each(shared_conf_list, _foreach_warn_unmatched_type, | |
| sharing_conf_list); | |
| while ((sharing_record = list_pop(sharing_conf_list))) { | |
| typed_record = list_find_first(shared_conf_list, | |
| _find_shared_by_sharing_type, | |
| sharing_record->type_name); | |
| /* | |
| * A node mixing sharded and unsharded gpu models must not | |
| * report any zero-count gres. | |
| */ | |
| if (typed_record) { | |
| shared_record = | |
| _create_shared_rec(sharing_record, shared_name, | |
| typed_record); | |
| shared_record->count = typed_record->count; | |
| list_append(gres_conf_list, shared_record); | |
| } | |
| list_append(gres_conf_list, sharing_record); | |
| } | |
| } | |
| static int _delete_leftovers(void *x, void *arg) | |
| { | |
| gres_slurmd_conf_t *gres_slurmd_conf = x; | |
| error("Discarding gres/'shared' configuration (File=%s) without matching gres/'sharing' record", | |
| gres_slurmd_conf->file); | |
| return 1; | |
| } | |
| /* | |
| * Merge SHARED records back to the original list, updating and reordering | |
| * as needed. | |
| */ | |
| static int _merge_lists(list_t *gres_conf_list, list_t *sharing_conf_list, | |
| list_t *shared_conf_list, char *shared_name) | |
| { | |
| gres_slurmd_conf_t *sharing_record, *shared_record; | |
| /* | |
| * Shard or MPS with no GPU to put it on. Drop it and continue. | |
| * slurmd must still register: refusing to start would leave the | |
| * controller advertising the previous shard layout on devices that | |
| * are gone. slurmctld has no device files for a cloud node yet, and | |
| * fatal() here would kill it. The node's slurmd sends the real | |
| * devices when it registers. A node whose Gres= asked for these | |
| * shards and got none is short, so it does not look unconfigured. | |
| */ | |
| if (!list_count(sharing_conf_list) && list_count(shared_conf_list)) { | |
| error("No sharing devices found; omitting gres/%s", | |
| shared_name); | |
| list_flush(shared_conf_list); | |
| return SLURM_SUCCESS; | |
| } | |
| /* | |
| * If a shared gres (mps or shard) has a Count, but no File | |
| * specification, then give each sharing device (gpu) of a matching | |
| * Type, a per-device Count. For entries without a Type, evenly | |
| * distribute Count over all gres sharing file records (gpus). | |
| * | |
| * _build_shared_list() guarantees records without File either all have | |
| * a Type or there is a single untyped one. | |
| * | |
| * Generated typed records hold a node total and distribute like | |
| * untyped ones. | |
| */ | |
| if (list_count(shared_conf_list)) { | |
| shared_record = list_peek(shared_conf_list); | |
| if (!shared_record->file) { | |
| if (shared_record->type_name && | |
| !(shared_record->config_flags & | |
| GRES_CONF_GENERATED)) | |
| _distribute_count_per_type(gres_conf_list, | |
| sharing_conf_list, | |
| shared_conf_list, | |
| shared_name); | |
| else | |
| _distribute_count(gres_conf_list, | |
| sharing_conf_list, | |
| shared_record->count, | |
| shared_record); | |
| list_flush(shared_conf_list); | |
| return SLURM_SUCCESS; | |
| } | |
| } | |
| /* | |
| * Add SHARED records, matching File ordering to that of SHARING | |
| * records | |
| */ | |
| while ((sharing_record = list_pop(sharing_conf_list))) { | |
| shared_record = list_remove_first(shared_conf_list, | |
| _find_matching_file_gres, | |
| sharing_record); | |
| if (shared_record) { | |
| /* | |
| * Copy gres/sharing Type & CPU info to | |
| * gres/shared | |
| */ | |
| if (sharing_record->type_name) { | |
| shared_record->config_flags |= | |
| GRES_CONF_HAS_TYPE; | |
| } | |
| if (sharing_record->cpus) { | |
| xfree(shared_record->cpus); | |
| shared_record->cpus = | |
| xstrdup(sharing_record->cpus); | |
| } | |
| if (sharing_record->cpus_bitmap) { | |
| shared_record->cpu_cnt = | |
| sharing_record->cpu_cnt; | |
| FREE_NULL_BITMAP( | |
| shared_record->cpus_bitmap); | |
| shared_record->cpus_bitmap = | |
| bit_copy(sharing_record-> | |
| cpus_bitmap); | |
| } | |
| xfree(shared_record->type_name); | |
| shared_record->type_name = | |
| xstrdup(sharing_record->type_name); | |
| /* | |
| * gres/shared has no device of its own, so it | |
| * consumes gres/sharing's UUID. Adopt it only if | |
| * gres/shared didn't already configure its own; a | |
| * configured value that disagrees with gres/sharing's | |
| * is rejected rather than silently discarded. | |
| */ | |
| if (!shared_record->unique_id) { | |
| shared_record->unique_id = | |
| xstrdup(sharing_record->unique_id); | |
| } else if (sharing_record->unique_id && | |
| xstrcmp(shared_record->unique_id, | |
| sharing_record->unique_id)) { | |
| fatal("Configured UUID=%s for gres/%s (File=%s) does not match UUID=%s of its gres/sharing device (File=%s)", | |
| shared_record->unique_id, shared_name, | |
| shared_record->file, | |
| sharing_record->unique_id, | |
| sharing_record->file); | |
| } | |
| list_append(gres_conf_list, shared_record); | |
| } else { | |
| /* Add gres/shared record to match gres/gps record */ | |
| shared_record = _create_shared_rec( | |
| sharing_record, shared_name, NULL); | |
| shared_record->count = 0; | |
| list_append(gres_conf_list, shared_record); | |
| } | |
| list_append(gres_conf_list, sharing_record); | |
| } | |
| /* Remove any remaining SHARED records (no matching File) */ | |
| (void) list_delete_all(shared_conf_list, _delete_leftovers, NULL); | |
| return SLURM_SUCCESS; | |
| } | |
| /* Translate device file name to numeric index "/dev/nvidia2" -> 2 */ | |
| static int _compute_local_id(char *dev_file_name) | |
| { | |
| int i, local_id = -1, mult = 1; | |
| if (!dev_file_name) | |
| return -1; | |
| for (i = strlen(dev_file_name) - 1; i >= 0; i--) { | |
| if ((dev_file_name[i] < '0') || (dev_file_name[i] > '9')) | |
| break; | |
| if (local_id == -1) | |
| local_id = 0; | |
| local_id += (dev_file_name[i] - '0') * mult; | |
| mult *= 10; | |
| } | |
| return local_id; | |
| } | |
| static int _find_dev_id(void *x, void *arg) | |
| { | |
| shared_dev_info_t *shared = x; | |
| int *id = arg; | |
| return (shared->id == *id); | |
| } | |
| /* | |
| * Collect the count of every device of one shared gres, keyed by the number at | |
| * the end of its device file name. | |
| * IN gres_conf_list - gres.conf records of the node | |
| * IN plugin_id - shared gres to collect, the one of this plugin | |
| * RET total count of the shared gres on the node | |
| */ | |
| static uint64_t _build_shared_dev_info(list_t *gres_conf_list, | |
| uint32_t plugin_id) | |
| { | |
| uint64_t shared_count = 0; | |
| gres_slurmd_conf_t *gres_slurmd_conf; | |
| shared_dev_info_t *shared_conf; | |
| list_itr_t *iter; | |
| FREE_NULL_LIST(shared_info); | |
| shared_info = list_create(xfree_ptr); | |
| iter = list_iterator_create(gres_conf_list); | |
| while ((gres_slurmd_conf = list_next(iter))) { | |
| /* | |
| * Every shared gres of the node reaches this list, and each | |
| * shared plugin holds its own copy of it, so the records of | |
| * the other shared gres are not this plugin's to collect. | |
| */ | |
| if (!gres_id_shared(gres_slurmd_conf->config_flags) || | |
| (gres_slurmd_conf->plugin_id != plugin_id)) | |
| continue; | |
| shared_conf = xmalloc(sizeof(shared_dev_info_t)); | |
| shared_conf->count = gres_slurmd_conf->count; | |
| shared_conf->id = _compute_local_id(gres_slurmd_conf->file); | |
| if (list_find_first(shared_info, _find_dev_id, | |
| &shared_conf->id)) | |
| error("gres.conf: %s device %s ends in the same number as another one of the node, so the count of neither can be told from the other. Give the devices of a node file names that end in different numbers.", | |
| gres_slurmd_conf->name, gres_slurmd_conf->file); | |
| log_flag(GRES, "%s: %s id=%d count=%"PRIu64, | |
| __func__, gres_slurmd_conf->file, shared_conf->id, | |
| shared_conf->count); | |
| list_append(shared_info, shared_conf); | |
| shared_count += gres_slurmd_conf->count; | |
| } | |
| list_iterator_destroy(iter); | |
| return shared_count; | |
| } | |
| /* | |
| * Count of gres/shared records is zero, remove them from GRES list sent to | |
| * slurmctld daemon. | |
| */ | |
| static int _remove_shared_recs(void *x, void *arg) | |
| { | |
| gres_slurmd_conf_t *gres_slurmd_conf = x; | |
| char *shared_name = arg; | |
| if (!xstrcmp(gres_slurmd_conf->name, shared_name)) | |
| return 1; | |
| return 0; | |
| } | |
| /* | |
| * Convert all sharing records to a new entries in a list where each File is a | |
| * unique device (i.e. convert a record with "File=nvidia[0-3]" into 4 separate | |
| * records). | |
| */ | |
| static list_t *_build_sharing_list(list_t *gres_list, char *sharing_name) | |
| { | |
| list_itr_t *itr; | |
| gres_slurmd_conf_t *gres_slurmd_conf, *sharing_record; | |
| list_t *sharing_list; | |
| hostlist_t *hl; | |
| char *f_name; | |
| bool log_fname = true; | |
| if (gres_list == NULL) | |
| return NULL; | |
| sharing_list = list_create(destroy_gres_slurmd_conf); | |
| itr = list_iterator_create(gres_list); | |
| while ((gres_slurmd_conf = list_next(itr))) { | |
| if (xstrcmp(gres_slurmd_conf->name, sharing_name)) | |
| continue; | |
| if (!gres_slurmd_conf->file) { | |
| if (log_fname) { | |
| error("SHARING configuration lacks \"File\" specification"); | |
| log_fname = false; | |
| } | |
| continue; | |
| } | |
| /* | |
| * Do not split up gres records with MultipleFiles, e.g. MIGs. | |
| * With MultipleFiles, all files correspond to the same gres | |
| * device, as opposed to File=nvidia[0-3] which corresponds to | |
| * four separate gres devices. | |
| */ | |
| if (gres_slurmd_conf->config_flags & GRES_CONF_HAS_MULT) { | |
| list_append(sharing_list, gres_slurmd_conf); | |
| list_remove(itr); | |
| continue; | |
| } | |
| hl = hostlist_create(gres_slurmd_conf->file); | |
| while ((f_name = hostlist_shift(hl))) { | |
| sharing_record = xmalloc(sizeof(gres_slurmd_conf_t)); | |
| sharing_record->chunk_order = | |
| xstrdup(gres_slurmd_conf->chunk_order); | |
| sharing_record->chunk_sizes = | |
| xstrdup(gres_slurmd_conf->chunk_sizes); | |
| sharing_record->config_flags = | |
| gres_slurmd_conf->config_flags; | |
| if (gres_slurmd_conf->type_name) { | |
| sharing_record->config_flags |= | |
| GRES_CONF_HAS_TYPE; | |
| } | |
| sharing_record->count = 1; | |
| sharing_record->cpu_cnt = gres_slurmd_conf->cpu_cnt; | |
| sharing_record->cpus = xstrdup(gres_slurmd_conf->cpus); | |
| if (gres_slurmd_conf->cpus_bitmap) { | |
| sharing_record->cpus_bitmap = | |
| bit_copy(gres_slurmd_conf->cpus_bitmap); | |
| } | |
| sharing_record->dmem_region = | |
| xstrdup(gres_slurmd_conf->dmem_region); | |
| sharing_record->file = f_name; | |
| sharing_record->links = | |
| xstrdup(gres_slurmd_conf->links); | |
| sharing_record->name = xstrdup(gres_slurmd_conf->name); | |
| sharing_record->pci_addr = | |
| xstrdup(gres_slurmd_conf->pci_addr); | |
| sharing_record->plugin_id = gres_slurmd_conf->plugin_id; | |
| sharing_record->type_name = | |
| xstrdup(gres_slurmd_conf->type_name); | |
| sharing_record->unique_id = | |
| xstrdup(gres_slurmd_conf->unique_id); | |
| list_append(sharing_list, sharing_record); | |
| } | |
| hostlist_destroy(hl); | |
| (void) list_delete_item(itr); | |
| } | |
| list_iterator_destroy(itr); | |
| return sharing_list; | |
| } | |
| /* | |
| * Convert all shared_name records to a new entries in a list where each File is | |
| * a unique device (i.e. convert a record with "File=nvidia[0-3]" into 4 | |
| * separate records). Similar to _build_gpu_list(), but we copy more fields, | |
| * divide the "Count" across all shared_name records and remove from the | |
| * original list. | |
| */ | |
| static list_t *_build_shared_list(list_t *gres_list, char *shared_name) | |
| { | |
| list_itr_t *itr; | |
| gres_slurmd_conf_t *gres_slurmd_conf, *shared_record; | |
| list_t *shared_list; | |
| hostlist_t *hl; | |
| char *f_name; | |
| uint64_t count_per_file; | |
| int shared_no_file_recs = 0, shared_file_recs = 0; | |
| int shared_typed_recs = 0; | |
| if (gres_list == NULL) | |
| return NULL; | |
| shared_list = list_create(destroy_gres_slurmd_conf); | |
| itr = list_iterator_create(gres_list); | |
| while ((gres_slurmd_conf = list_next(itr))) { | |
| if (xstrcmp(gres_slurmd_conf->name, shared_name)) | |
| continue; | |
| if (!gres_slurmd_conf->file) { | |
| if (shared_file_recs) | |
| fatal("%s: multiple configurations with and without \"File\"", | |
| __func__); | |
| if (gres_slurmd_conf->type_name && | |
| !(gres_slurmd_conf->config_flags & | |
| GRES_CONF_GENERATED)) { | |
| /* | |
| * Typed configurations without File give | |
| * their Count to every sharing device of a | |
| * matching Type. Generated records hold a | |
| * node total and are excluded. | |
| */ | |
| if (shared_no_file_recs) | |
| fatal("%s: bad configuration, mixing configurations with and without \"Type\" and no \"File\"", | |
| __func__); | |
| if (list_find_first(shared_list, | |
| _find_same_type_gres, | |
| gres_slurmd_conf)) | |
| fatal("%s: bad configuration, duplicate \"Type=%s\" configurations without \"File\"", | |
| __func__, | |
| gres_slurmd_conf->type_name); | |
| shared_typed_recs++; | |
| } else { | |
| if (shared_no_file_recs) | |
| fatal("%s: bad configuration, multiple configurations without \"File\"", | |
| __func__); | |
| if (shared_typed_recs) | |
| fatal("%s: bad configuration, mixing configurations with and without \"Type\" and no \"File\"", | |
| __func__); | |
| shared_no_file_recs++; | |
| } | |
| shared_record = xmalloc(sizeof(gres_slurmd_conf_t)); | |
| shared_record->config_flags = | |
| gres_slurmd_conf->config_flags; | |
| if (gres_slurmd_conf->type_name) | |
| shared_record->config_flags |= GRES_CONF_HAS_TYPE; | |
| shared_record->count = gres_slurmd_conf->count; | |
| shared_record->cpu_cnt = gres_slurmd_conf->cpu_cnt; | |
| shared_record->cpus = xstrdup(gres_slurmd_conf->cpus); | |
| if (gres_slurmd_conf->cpus_bitmap) { | |
| shared_record->cpus_bitmap = | |
| bit_copy(gres_slurmd_conf->cpus_bitmap); | |
| } | |
| shared_record->name = xstrdup(gres_slurmd_conf->name); | |
| shared_record->plugin_id = gres_slurmd_conf->plugin_id; | |
| shared_record->type_name = | |
| xstrdup(gres_slurmd_conf->type_name); | |
| shared_record->unique_id = | |
| xstrdup(gres_slurmd_conf->unique_id); | |
| list_append(shared_list, shared_record); | |
| } else { | |
| shared_file_recs++; | |
| if (shared_no_file_recs || shared_typed_recs) | |
| fatal("gres/shared: multiple configurations with and without \"File\""); | |
| hl = hostlist_create(gres_slurmd_conf->file); | |
| count_per_file = | |
| gres_slurmd_conf->count / hostlist_count(hl); | |
| while ((f_name = hostlist_shift(hl))) { | |
| shared_record = | |
| xmalloc(sizeof(gres_slurmd_conf_t)); | |
| shared_record->chunk_order = | |
| xstrdup(gres_slurmd_conf->chunk_order); | |
| shared_record->chunk_sizes = | |
| xstrdup(gres_slurmd_conf->chunk_sizes); | |
| shared_record->config_flags = | |
| gres_slurmd_conf->config_flags; | |
| if (gres_slurmd_conf->type_name) { | |
| shared_record->config_flags |= | |
| GRES_CONF_HAS_TYPE; | |
| } | |
| shared_record->count = count_per_file; | |
| shared_record->cpu_cnt = | |
| gres_slurmd_conf->cpu_cnt; | |
| shared_record->cpus = xstrdup( | |
| gres_slurmd_conf->cpus); | |
| if (gres_slurmd_conf->cpus_bitmap) { | |
| shared_record->cpus_bitmap = | |
| bit_copy(gres_slurmd_conf-> | |
| cpus_bitmap); | |
| } | |
| shared_record->file = f_name; | |
| shared_record->name = xstrdup( | |
| gres_slurmd_conf->name); | |
| shared_record->plugin_id = | |
| gres_slurmd_conf->plugin_id; | |
| shared_record->type_name = | |
| xstrdup(gres_slurmd_conf->type_name); | |
| shared_record->unique_id = | |
| xstrdup(gres_slurmd_conf->unique_id); | |
| list_append(shared_list, shared_record); | |
| } | |
| hostlist_destroy(hl); | |
| } | |
| (void) list_delete_item(itr); | |
| } | |
| list_iterator_destroy(itr); | |
| return shared_list; | |
| } | |
| extern void gres_c_s_fini(void) | |
| { | |
| FREE_NULL_LIST(shared_info); | |
| } | |
| /* | |
| * We could load gres state or validate it using various mechanisms here. | |
| * This only validates that the configuration was specified in gres.conf. | |
| * In the general case, no code would need to be changed. | |
| */ | |
| extern int gres_c_s_init_share_devices(list_t *gres_conf_list, | |
| list_t **share_devices, | |
| node_config_load_t *config, | |
| char *sharing_name) | |
| { | |
| int rc = SLURM_SUCCESS; | |
| list_t *sharing_conf_list, *shared_conf_list; | |
| log_level_t log_lvl; | |
| if (slurm_conf.debug_flags & DEBUG_FLAG_GRES) | |
| log_lvl = LOG_LEVEL_VERBOSE; | |
| else | |
| log_lvl = LOG_LEVEL_DEBUG; | |
| /* Assume this state is caused by an scontrol reconfigure */ | |
| if (share_devices && *share_devices) { | |
| debug("Resetting share_devices"); | |
| FREE_NULL_LIST(*share_devices); | |
| } | |
| log_flag(GRES, "Initialized gres.conf list"); | |
| print_gres_list(gres_conf_list, log_lvl); | |
| /* | |
| * Ensure that every SHARING device file is listed as a SHARED file. | |
| * Any SHARED entry that we need to add will have a "Count" of zero. | |
| * Every SHARED "Type" will be made to match the SHARING "Type". The | |
| * order of SHARED records (by "File") must match the order in which | |
| * SHARINGs are defined for the GRES bitmaps in slurmctld to line up. | |
| * | |
| * First, convert all SHARING records to a new entries in a list where | |
| * each File is a unique device (i.e. convert a record with | |
| * "File=nvidia[0-3]" into 4 separate records). | |
| */ | |
| sharing_conf_list = _build_sharing_list( | |
| gres_conf_list, sharing_name); | |
| /* Now move SHARED records to new list, each with unique device file */ | |
| shared_conf_list = _build_shared_list( | |
| gres_conf_list, config->gres_name); | |
| /* | |
| * Merge SHARED records back to original list, updating and reordering | |
| * as needed. | |
| */ | |
| rc = _merge_lists(gres_conf_list, sharing_conf_list, shared_conf_list, | |
| config->gres_name); | |
| FREE_NULL_LIST(sharing_conf_list); | |
| FREE_NULL_LIST(shared_conf_list); | |
| if (rc != SLURM_SUCCESS) | |
| fatal("failed to merge SHARED and SHARING configuration"); | |
| rc = gres_node_config_load(gres_conf_list, config, share_devices); | |
| if (rc != SLURM_SUCCESS) | |
| fatal("failed to load configuration"); | |
| if (!_build_shared_dev_info(gres_conf_list, | |
| gres_build_id(config->gres_name)) && | |
| gres_conf_list) | |
| (void) list_delete_all(gres_conf_list, _remove_shared_recs, | |
| config->gres_name); | |
| log_var(log_lvl, "Final gres.conf list:"); | |
| print_gres_list(gres_conf_list, log_lvl); | |
| return rc; | |
| } | |
| extern void gres_c_s_send_stepd(buf_t *buffer) | |
| { | |
| uint32_t shared_cnt; | |
| shared_dev_info_t *shared_ptr; | |
| list_itr_t *itr; | |
| if (!shared_info) { | |
| shared_cnt = 0; | |
| pack32(shared_cnt, buffer); | |
| } else { | |
| shared_cnt = list_count(shared_info); | |
| pack32(shared_cnt, buffer); | |
| itr = list_iterator_create(shared_info); | |
| while ((shared_ptr = list_next(itr))) { | |
| pack64(shared_ptr->count, buffer); | |
| pack64(shared_ptr->id, buffer); | |
| } | |
| list_iterator_destroy(itr); | |
| } | |
| return; | |
| } | |
| /* Receive GRES information from slurmd on the specified file descriptor */ | |
| extern void gres_c_s_recv_stepd(buf_t *buffer) | |
| { | |
| shared_dev_info_t *shared_ptr = NULL; | |
| uint64_t uint64_tmp; | |
| uint32_t shared_cnt; | |
| safe_unpack32(&shared_cnt, buffer); | |
| if (!shared_cnt) | |
| return; | |
| FREE_NULL_LIST(shared_info); | |
| shared_info = list_create(xfree_ptr); | |
| for (uint32_t i = 0; i < shared_cnt; i++) { | |
| shared_ptr = xmalloc(sizeof(shared_dev_info_t)); | |
| safe_unpack64(&uint64_tmp, buffer); | |
| shared_ptr->count = uint64_tmp; | |
| safe_unpack64(&uint64_tmp, buffer); | |
| shared_ptr->id = uint64_tmp; | |
| list_append(shared_info, shared_ptr); | |
| } | |
| return; | |
| unpack_error: | |
| error("failed"); | |
| xfree(shared_ptr); | |
| return; | |
| } |