blob: 70d24cd8c628ba61ee4930f28fc8f160a989b716 [file]
/*****************************************************************************\
* Copyright (C) 2026 Google LLC.
\*****************************************************************************/
/*
* Unit test of the gres/tpu sysfs discovery through gres_g_node_config_load():
* how the PCI functions of a mock /sys/bus/pci/devices tree are grouped into
* one GRES record per TPU chip, which ones are ignored, and the order of the
* chips in the final list of gres_slurmd_conf_t records.
*
* Every case builds its own mock sysfs tree (pointed to by
* SLURM_TPU_SYSFS_PATH), slurm.conf, tpu.yaml and gres.conf. Unless a case
* gives the node GRES of slurm.conf, the node has none, so every TPU record
* comes from the detection. Note that without node GRES the gres.conf records
* are dropped before the plugins run, as slurm.conf sets the count of each
* GRES of the node.
*
* The mock tree has 4 chips, in PCI slots 08 to 05, each with 1 or 2 PCI
* functions (cores). IOMMU groups start at 8, so they span single and double
* digits (a lexicographic sort differs from a numeric one), and they are given
* in reverse slot order (sorting by address cannot pass for sorting by group).
* It also has 3 functions that must be skipped:
* - 0000:00:09.0: matching vendor/device with no IOMMU group
* - 0000:00:0a.0: vendor 0x1ae0 with an unconfigured PCI device ID (0x00ff)
* - 0000:00:0b.0: matching PCI device ID with a non-matching vendor (0x9999)
*
* The final list also keeps the record of the other GRES (fake) of GresTypes.
*
* Originally test_144_40 (python).
*/
#include <check.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include "slurm/slurm.h"
#include "src/common/list.h"
#include "src/common/log.h"
#include "src/common/macros.h"
#include "src/common/read_config.h"
#include "src/common/xmalloc.h"
#include "src/common/xstring.h"
#include "src/interfaces/gres.h"
#include "scheck.h"
#define NODE_NAME "testnode"
#define CPU_CNT 96
#define CPULIST "0-95"
#define LOAD_LOG "load.log"
#define TPU_VENDOR "0x1ae0"
#define V6E_ID "0x006f"
#define TPU7X_ID "0x0076"
#define MAX_CHIPS 4
#define MAX_EXTRA 4
#define SLURM_CONF_CONTENT \
"ControlMachine=test_machine\n" \
"ClusterName=test_cluster\n" \
"SlurmUser=root\n" \
"GresTypes=tpu,fake\n"
#define TPU_YAML_CONTENT \
"tpu_types:\n" \
" - type: tpu7x\n" \
" pci_device_id: \"" TPU7X_ID "\"\n" \
" shape_per_node: 2x2x1\n" \
" shape_per_segment: 4x4x4\n" \
" nodes_per_pod: 2304\n" \
" - type: v6e\n" \
" pci_device_id: \"" V6E_ID "\"\n" \
" shape_per_node: 2x2\n" \
" shape_per_segment: 16x16\n" \
" nodes_per_pod: 64\n"
#define NO_IOMMU_REGEX "has no IOMMU group"
#define SHARED_REGEX "shares an IOMMU group"
#define V6E_FILES \
{ "/dev/vfio/8", "/dev/vfio/9", "/dev/vfio/10", "/dev/vfio/11" }
#define TPU7X_FILES \
{ "/dev/vfio/8,/dev/vfio/9", "/dev/vfio/10,/dev/vfio/11", \
"/dev/vfio/12,/dev/vfio/13", "/dev/vfio/14,/dev/vfio/15" }
/* One more PCI function of the device_id of the case */
typedef struct {
char *addr;
int group;
} extra_func_t;
typedef struct {
char *id;
char *device_id; /* PCI device ID of the chips, NULL for no devices */
int cores; /* PCI functions per chip */
bool long_cpulist; /* local_cpulist longer than 255 bytes */
extra_func_t extra[MAX_EXTRA];
char *node_gres; /* GRES of the node in slurm.conf */
char *gres_conf;
/* Expected TPU records */
char *type;
char *files[MAX_CHIPS + 1]; /* NULL terminated, in order */
bool mult; /* chips with more than one core */
uint64_t fake_cnt; /* Count of the fake GRES record */
/* If set, log_cnt log lines must match this regex */
char *log_regex;
int log_cnt;
} tpu_case_t;
// clang-format off
static const tpu_case_t cases[] = {
/* Single-core chips (v6e) have 1 VFIO file per record */
{
.id = "v6e_single_core_discovery",
.device_id = V6E_ID,
.cores = 1,
.gres_conf = "AutoDetect=tpu\n",
.type = "v6e",
.files = V6E_FILES,
.log_regex = NO_IOMMU_REGEX,
.log_cnt = 1,
},
/*
* Dual-core chips (tpu7x) group their 2 PCI functions in one record,
* sorted numerically by their lowest IOMMU group
*/
{
.id = "tpu7x_dual_core_grouping_and_numeric_iommu_order",
.device_id = TPU7X_ID,
.cores = 2,
.gres_conf = "AutoDetect=tpu\n",
.type = "tpu7x",
.files = TPU7X_FILES,
.mult = true,
},
/* A local_cpulist longer than 255 bytes is read in full */
{
.id = "long_local_cpulist",
.device_id = V6E_ID,
.cores = 1,
.long_cpulist = true,
.gres_conf = "AutoDetect=tpu\n",
.type = "v6e",
.files = V6E_FILES,
},
/* A node with no TPU PCI devices has no TPU records */
{
.id = "empty_sysfs_discovers_nothing",
.gres_conf = "AutoDetect=tpu\n",
},
/* AutoDetect=off in gres.conf suppresses the sysfs scan */
{
.id = "autodetect_off_suppresses_sysfs_scan",
.device_id = TPU7X_ID,
.cores = 2,
.gres_conf = "AutoDetect=off\n",
.log_regex = "AutoDetect=off, not scanning for TPUs",
.log_cnt = 1,
},
/* Detection runs with AutoDetect unset */
{
.id = "autodetect_runs_unless_off_unset",
.device_id = V6E_ID,
.cores = 1,
.gres_conf = "",
.type = "v6e",
.files = V6E_FILES,
},
/* Detection runs with a node's AutoDetect=tpu over a global off */
{
.id = "autodetect_runs_unless_off_node_overrides_global_off",
.device_id = V6E_ID,
.cores = 1,
.gres_conf = "AutoDetect=off\n"
"NodeName=" NODE_NAME " AutoDetect=tpu\n",
.type = "v6e",
.files = V6E_FILES,
},
/*
* Detected chips replace the Name=tpu lines of gres.conf, and the
* lines of other GRES are kept. The node GRES keep the gres.conf lines
* until the plugin runs.
*/
{
.id = "detected_chips_replace_configured_tpu_lines",
.device_id = TPU7X_ID,
.cores = 2,
.node_gres = "tpu:tpu7x:2,fake:1",
.gres_conf = "Name=tpu Type=tpu7x Count=2\n"
"Name=fake Count=1\n",
.type = "tpu7x",
.files = TPU7X_FILES,
.mult = true,
.fake_cnt = 1,
},
/* Single-core chips that share an IOMMU group are ignored */
{
.id = "chips_sharing_an_iommu_group_ignored_single_core",
.device_id = V6E_ID,
.cores = 1,
.extra = {
{ "0000:00:0c.0", 20 },
{ "0000:00:0d.0", 20 },
},
.gres_conf = "AutoDetect=tpu\n",
.type = "v6e",
.files = V6E_FILES,
.log_regex = SHARED_REGEX,
.log_cnt = 2,
},
/* Dual-core chips that share an IOMMU group are ignored */
{
.id = "chips_sharing_an_iommu_group_ignored_dual_core",
.device_id = TPU7X_ID,
.cores = 2,
.extra = {
{ "0000:00:0c.0", 20 },
{ "0000:00:0c.1", 21 },
{ "0000:00:0d.0", 21 },
{ "0000:00:0d.1", 22 },
},
.gres_conf = "AutoDetect=tpu\n",
.type = "tpu7x",
.files = TPU7X_FILES,
.mult = true,
.log_regex = SHARED_REGEX,
.log_cnt = 2,
},
};
// clang-format on
static char *base_dir = NULL;
/* RET a local_cpulist of all the CPUs that is longer than 255 bytes */
static char *_long_cpulist(void)
{
char *cpulist = NULL;
for (int i = 0; i < CPU_CNT; i++)
xstrfmtcat(cpulist, "%s%d", (i ? "," : ""), i);
ck_assert_msg(strlen(cpulist) > 255,
"The cpulist should be long enough to truncate");
return cpulist;
}
static void _mkdir(const char *path)
{
ck_assert_msg(!mkdir(path, 0700) || (errno == EEXIST),
"mkdir(%s) failed: %s", path, strerror(errno));
}
/*
* Add the PCI function addr to the mock sysfs tree, in the IOMMU group group
* (none if negative).
*/
static void _make_func(const char *sysfs_dir, const char *addr,
const char *vendor, const char *device_id, int group,
const char *cpulist)
{
char *dev_dir = NULL, *content = NULL;
xstrfmtcat(dev_dir, "%s/%s", sysfs_dir, addr);
_mkdir(dev_dir);
xstrfmtcat(content, "%s\n", vendor);
scheck_write_file(dev_dir, "vendor", content);
xfree(content);
xstrfmtcat(content, "%s\n", device_id);
scheck_write_file(dev_dir, "device", content);
xfree(content);
xstrfmtcat(content, "%s\n", cpulist);
scheck_write_file(dev_dir, "local_cpulist", content);
xfree(content);
if (group >= 0) {
char *group_dir = NULL, *link = NULL, *target = NULL;
xstrfmtcat(group_dir, "%s/.groups/%d", sysfs_dir, group);
_mkdir(group_dir);
xstrfmtcat(link, "%s/iommu_group", dev_dir);
xstrfmtcat(target, "../.groups/%d", group);
ck_assert_msg(!symlink(target, link), "symlink(%s) failed: %s",
link, strerror(errno));
xfree(group_dir);
xfree(link);
xfree(target);
}
xfree(dev_dir);
}
/*
* Build the mock sysfs tree of a case in sysfs_dir. See the description of the
* tree at the top.
*/
static void _build_sysfs(const tpu_case_t *tc, const char *sysfs_dir,
const char *cpulist)
{
const char *slots[MAX_CHIPS] = { "08", "07", "06", "05" };
char *groups_dir = NULL;
int group = 8;
_mkdir(sysfs_dir);
xstrfmtcat(groups_dir, "%s/.groups", sysfs_dir);
_mkdir(groups_dir);
xfree(groups_dir);
if (!tc->device_id)
return;
for (int i = 0; i < MAX_CHIPS; i++) {
for (int core = 0; core < tc->cores; core++) {
char *addr =
xstrdup_printf("0000:00:%s.%d", slots[i], core);
_make_func(sysfs_dir, addr, TPU_VENDOR, tc->device_id,
group++, cpulist);
xfree(addr);
}
}
_make_func(sysfs_dir, "0000:00:09.0", TPU_VENDOR, tc->device_id, -1,
cpulist);
_make_func(sysfs_dir, "0000:00:0a.0", TPU_VENDOR, "0x00ff", 99,
cpulist);
_make_func(sysfs_dir, "0000:00:0b.0", "0x9999", tc->device_id, 98,
cpulist);
for (int i = 0; (i < MAX_EXTRA) && tc->extra[i].addr; i++)
_make_func(sysfs_dir, tc->extra[i].addr, TPU_VENDOR,
tc->device_id, tc->extra[i].group, cpulist);
}
/*
* Create the configuration and the mock sysfs tree of a case in its own
* directory, and point SLURM_CONF and SLURM_TPU_SYSFS_PATH to them.
* RET the case directory
*/
static char *_setup_case(const tpu_case_t *tc, const char *cpulist)
{
char *case_dir = NULL, *path = NULL;
xstrfmtcat(case_dir, "%s/%s", base_dir, tc->id);
_mkdir(case_dir);
scheck_write_file(case_dir, "slurm.conf", SLURM_CONF_CONTENT);
scheck_write_file(case_dir, "tpu.yaml", TPU_YAML_CONTENT);
scheck_write_file(case_dir, "gres.conf", tc->gres_conf);
xstrfmtcat(path, "%s/sysfs", case_dir);
_build_sysfs(tc, path, cpulist);
setenv("SLURM_TPU_SYSFS_PATH", path, 1);
xfree(path);
xstrfmtcat(path, "%s/slurm.conf", case_dir);
setenv("SLURM_CONF", path, 1);
xfree(path);
return case_dir;
}
/* sysfs reports machine CPU numbering, keep it as the abstract one */
static int _mac_to_abs(char *mac, char **abs)
{
*abs = xstrdup(mac);
return SLURM_SUCCESS;
}
static char *_recs2str(list_t *conf_list)
{
list_itr_t *itr = NULL;
gres_slurmd_conf_t *rec = NULL;
char *str = NULL;
if (!conf_list)
return xstrdup("(none)");
itr = list_iterator_create(conf_list);
while ((rec = list_next(itr)))
xstrfmtcat(str,
"\n Name=%s Type=%s Count=%" PRIu64
" Cores=%s File=%s Flags=%s",
rec->name, rec->type_name, rec->count, rec->cpus,
rec->file, gres_flags2str(rec->config_flags));
list_iterator_destroy(itr);
return str;
}
static void _check_str(const char *id, int idx, const char *field,
const char *exp, const char *act, const char *recs)
{
ck_assert_msg(
!xstrcmp(exp, act),
"%s: TPU record %d %s should be %s, but got %s. Records:%s", id,
idx, field, exp ? exp : "(null)", act ? act : "(null)", recs);
}
/*
* Check the TPU records (name tpu with a non-zero count, as a GRES of
* GresTypes without any record gets one with a zero count) and the record of
* the fake GRES.
*/
static void _check_records(const tpu_case_t *tc, const char *cpulist)
{
list_t *conf_list = gres_get_node_conf_list();
list_itr_t *itr = NULL;
gres_slurmd_conf_t *rec = NULL;
char *recs = _recs2str(conf_list);
int exp_cnt = 0, tpu_cnt = 0, fake_cnt = 0;
while ((exp_cnt < MAX_CHIPS) && tc->files[exp_cnt])
exp_cnt++;
if (conf_list)
itr = list_iterator_create(conf_list);
while (itr && (rec = list_next(itr))) {
if (!xstrcmp(rec->name, "fake")) {
ck_assert_msg(
rec->count == tc->fake_cnt,
"%s: the fake GRES Count should be %" PRIu64
", but got %" PRIu64 ". Records:%s",
tc->id, tc->fake_cnt, rec->count, recs);
fake_cnt++;
continue;
}
if (xstrcmp(rec->name, "tpu") || !rec->count)
continue;
ck_assert_msg(
tpu_cnt < exp_cnt,
"%s: there should be %d TPU records, but got more. Records:%s",
tc->id, exp_cnt, recs);
_check_str(tc->id, tpu_cnt, "Type", tc->type, rec->type_name,
recs);
_check_str(tc->id, tpu_cnt, "File", tc->files[tpu_cnt],
rec->file, recs);
_check_str(tc->id, tpu_cnt, "Cores", cpulist, rec->cpus, recs);
ck_assert_msg(
rec->count == 1,
"%s: TPU record %d Count should be 1, but got %" PRIu64
". Records:%s",
tc->id, tpu_cnt, rec->count, recs);
ck_assert_msg(
rec->config_flags & GRES_CONF_AUTODETECT,
"%s: TPU record %d should be autodetected. Records:%s",
tc->id, tpu_cnt, recs);
ck_assert_msg(
!(rec->config_flags & GRES_CONF_HAS_MULT) == !tc->mult,
"%s: TPU record %d should %sbe a multi-core chip. Records:%s",
tc->id, tpu_cnt, (tc->mult ? "" : "not "), recs);
tpu_cnt++;
}
if (itr)
list_iterator_destroy(itr);
ck_assert_msg(
tpu_cnt == exp_cnt,
"%s: there should be %d TPU records, but got %d. Records:%s",
tc->id, exp_cnt, tpu_cnt, recs);
ck_assert_msg(
fake_cnt == 1,
"%s: there should be 1 fake GRES record, but got %d. Records:%s",
tc->id, fake_cnt, recs);
FREE_NULL_LIST(conf_list);
xfree(recs);
}
START_TEST(test_tpu_discovery)
{
const tpu_case_t *tc = &cases[_i];
char *case_dir = NULL, *log_path = NULL, *log_txt = NULL;
char *cpulist = tc->long_cpulist ? _long_cpulist() : xstrdup(CPULIST);
list_t *node_gres = NULL;
int rc = SLURM_ERROR;
case_dir = _setup_case(tc, cpulist);
xstrfmtcat(log_path, "%s/%s", case_dir, LOAD_LOG);
/*
* Log the debug messages of the plugin too, and keep logging into
* stderr to show any unexpected fatal()
*/
ck_assert_msg(!scheck_log_to_file(log_path, LOG_LEVEL_DEBUG,
LOG_LEVEL_INFO),
"%s: unable to log into %s", tc->id, log_path);
slurm_init(NULL);
if (tc->node_gres)
gres_init_node_config(tc->node_gres, &node_gres);
rc = gres_g_node_config_load(CPU_CNT, NODE_NAME, node_gres, NULL,
_mac_to_abs);
FREE_NULL_LIST(node_gres);
log_txt = scheck_read_file(log_path);
ck_assert_msg(
rc == SLURM_SUCCESS,
"%s: gres_g_node_config_load() should succeed, but got: %s. Log:\n%s",
tc->id, slurm_strerror(rc), log_txt);
_check_records(tc, cpulist);
if (tc->log_regex)
scheck_assert_log_count(tc->id, log_txt, tc->log_regex,
tc->log_cnt);
xfree(log_txt);
xfree(log_path);
xfree(case_dir);
xfree(cpulist);
}
END_TEST
int main(void)
{
SRunner *sr = NULL;
Suite *s = NULL;
TCase *tc_core = NULL;
int number_failed = 0;
log_options_t log_opts = LOG_OPTS_INITIALIZER;
log_opts.stderr_level = LOG_LEVEL_INFO;
log_init("gres-tpu-test", log_opts, 0, NULL);
/* Create the case directories in the working directory */
if (!(base_dir = scheck_mkdtemp("gres_tpu")))
return EXIT_FAILURE;
tc_core = tcase_create("tpu_discovery");
tcase_add_loop_test(tc_core, test_tpu_discovery, 0, ARRAY_SIZE(cases));
s = suite_create("gres_tpu");
suite_add_tcase(s, tc_core);
sr = srunner_create(s);
srunner_run_all(sr, CK_ENV);
number_failed = srunner_ntests_failed(sr);
srunner_free(sr);
xfree(base_dir);
log_fini();
return (number_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE;
}