Google Git
Sign in
third-party-mirror / SchedMD / slurm / 5ba844f540900017f593e9caf70005e0e009ad30 / . / src / slurmctld / node_scheduler.c
blob: 3886db79d6a4883acab86729d8d403cb9815bfe0 [file] [log] [blame]
/*****************************************************************************\
* node_scheduler.c - select and allocated nodes to jobs
* Note: there is a global node table (node_record_table_ptr)
*****************************************************************************
* Copyright (C) 2002-2007 The Regents of the University of California.
* Copyright (C) 2008-2010 Lawrence Livermore National Security.
* Copyright (C) SchedMD LLC.
* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
* Written by Morris Jette <jette1@llnl.gov>
* CODE-OCEC-09-009. All rights reserved.
*
* This file is part of Slurm, a resource management program.
* For details, see <https://slurm.schedmd.com/>.
* Please also read the included file: DISCLAIMER.
*
* Slurm is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License as published by the Free
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
*
* In addition, as a special exception, the copyright holders give permission
* to link the code of portions of this program with the OpenSSL library under
* certain conditions as described in each individual source file, and
* distribute linked combinations including the two. You must obey the GNU
* General Public License in all respects for all of the code used other than
* OpenSSL. If you modify file(s) with this exception, you may extend this
* exception to your version of the file(s), but you are not obligated to do
* so. If you do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source files in
* the program, then also delete it here.
*
* Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along
* with Slurm; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
\*****************************************************************************/
#include "config.h"
#include <errno.h>
#include <pthread.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <syslog.h>
#include <unistd.h>
#include "slurm/slurm_errno.h"
#include "src/common/assoc_mgr.h"
#include "src/common/group_cache.h"
#include "src/common/hostlist.h"
#include "src/common/id_util.h"
#include "src/common/job_features.h"
#include "src/common/list.h"
#include "src/common/node_features.h"
#include "src/common/port_mgr.h"
#include "src/common/slurm_protocol_pack.h"
#include "src/common/xassert.h"
#include "src/common/xmalloc.h"
#include "src/common/xstring.h"
#include "src/interfaces/accounting_storage.h"
#include "src/interfaces/burst_buffer.h"
#include "src/interfaces/gres.h"
#include "src/interfaces/jobcomp.h"
#include "src/interfaces/mcs.h"
#include "src/interfaces/node_features.h"
#include "src/interfaces/preempt.h"
#include "src/interfaces/priority.h"
#include "src/interfaces/select.h"
#include "src/interfaces/switch.h"
#include "src/interfaces/topology.h"
#include "src/slurmctld/acct_policy.h"
#include "src/slurmctld/agent.h"
#include "src/slurmctld/gang.h"
#include "src/slurmctld/job_scheduler.h"
#include "src/slurmctld/licenses.h"
#include "src/slurmctld/node_scheduler.h"
#include "src/slurmctld/power_save.h"
#include "src/slurmctld/proc_req.h"
#include "src/slurmctld/reservation.h"
#include "src/slurmctld/slurmctld.h"
#include "src/stepmgr/gres_stepmgr.h"
#include "src/stepmgr/stepmgr.h"
#define _DEBUG 0
#define MAX_FEATURES 64 /* max exclusive features "[fs1|fs2]"=2 */
struct node_set { /* set of nodes with same configuration */
uint16_t cpus_per_node; /* NOTE: This is the minimum count */
char *features; /* Node features */
bitstr_t *feature_bits; /* MORed feature's position */
uint32_t flags; /* See NODE_SET_* below */
bitstr_t *my_bitmap; /* Node bitmap */
uint32_t node_cnt; /* Node count */
uint32_t node_weight; /* Node weight */
uint64_t real_memory; /* Real memory on node */
uint64_t sched_weight; /* Scheduling weight, based upon
* node_weight and flags */
};
#define NODE_SET_NOFLAG SLURM_BIT(0)
#define NODE_SET_REBOOT SLURM_BIT(1)
#define NODE_SET_OUTSIDE_FLEX SLURM_BIT(2)
#define NODE_SET_POWER_DN SLURM_BIT(3)
#define NODE_SET_POWERING_UP SLURM_BIT(4)
enum {
IN_FL, /* Inside flex reservation */
OUT_FL, /* Outside flex reservation */
IN_FL_RE, /* Inside flex reservation + need reboot */
OUT_FL_NO_RE, /* Outside flex reservation + NO to need reboot */
OUT_FL_RE, /* Outside flex reservation + need reboot */
REBOOT, /* Needs reboot */
NM_TYPES /* Number of node types */
};
static int _build_node_list(job_record_t *job_ptr,
struct node_set **node_set_pptr,
int *node_set_size, char **err_msg,
bool test_only, bool can_reboot);
static bitstr_t *_find_grp_node_bitmap(job_record_t *job_ptr);
static bool _first_array_task(job_record_t *job_ptr);
static void _log_node_set(job_record_t *job_ptr,
struct node_set *node_set_ptr,
int node_set_size);
static int _match_feature(list_t *feature_list, bitstr_t **inactive_bitmap);
static int _nodes_in_sets(bitstr_t *req_bitmap,
struct node_set * node_set_ptr,
int node_set_size);
static int _pick_best_nodes(struct node_set *node_set_ptr,
int node_set_size, bitstr_t ** select_bitmap,
job_record_t *job_ptr, part_record_t *part_ptr,
uint32_t min_nodes, uint32_t max_nodes,
uint32_t req_nodes, bool test_only,
list_t *preemptee_candidates,
list_t **preemptee_job_list, bool has_xand,
resv_exc_t *resv_exc_ptr, bool resv_overlap);
static void _set_err_msg(bool cpus_ok, bool mem_ok, bool disk_ok,
bool job_mc_ok, char **err_msg);
static void _set_sched_weight(struct node_set *node_set_ptr);
static int _sort_node_set(const void *x, const void *y);
static bitstr_t *_valid_features(job_record_t *job_ptr,
config_record_t *config_ptr,
bool can_reboot, bitstr_t *reboot_bitmap);
/*
* _get_ntasks_per_core - Retrieve the value of ntasks_per_core from
* the given job_details record. If it wasn't set, return INFINITE16.
* Intended for use with the adjust_cpus_nppcu function.
*/
static uint16_t _get_ntasks_per_core(job_details_t *details)
{
if (details->mc_ptr)
return details->mc_ptr->ntasks_per_core;
else
return INFINITE16;
}
/*
* allocate_nodes - change state of specified nodes to NODE_STATE_ALLOCATED
* also claim required licenses and resources reserved by accounting
* policy association
* IN job_ptr - job being allocated resources
*/
extern void allocate_nodes(job_record_t *job_ptr)
{
node_record_t *node_ptr;
for (int i = 0; (node_ptr = next_node_bitmap(job_ptr->node_bitmap, &i));
i++) {
make_node_alloc(node_ptr, job_ptr);
}
node_mgr_make_node_blocked(job_ptr, true);
last_node_update = time(NULL);
license_job_get(job_ptr, false);
set_initial_job_alias_list(job_ptr);
}
extern void set_initial_job_alias_list(job_record_t *job_ptr)
{
node_record_t *node_ptr;
bool has_cloud = false, has_cloud_power_save = false;
bool has_dynamic_norm = false;
for (int i = 0; (node_ptr = next_node_bitmap(job_ptr->node_bitmap, &i));
i++) {
if (IS_NODE_DYNAMIC_FUTURE(node_ptr))
has_cloud = true;
if (IS_NODE_DYNAMIC_NORM(node_ptr)) {
/* Must set alias list as nodes won't exist in conf */
has_cloud = true;
has_dynamic_norm = true;
}
if (IS_NODE_CLOUD(node_ptr)) {
has_cloud = true;
if (IS_NODE_POWERED_DOWN(node_ptr) ||
IS_NODE_POWERING_UP(node_ptr))
has_cloud_power_save = true;
}
}
if (has_cloud) {
if (has_cloud_power_save &&
job_ptr->origin_cluster &&
xstrcmp(slurm_conf.cluster_name, job_ptr->origin_cluster)) {
/* Set TBD so remote srun will updated node_addrs */
job_ptr->alias_list = xstrdup("TBD");
job_ptr->wait_all_nodes = 1;
} else if (cloud_dns && !has_dynamic_norm) {
job_ptr->wait_all_nodes = 1;
} else if (has_cloud_power_save) {
job_ptr->alias_list = xstrdup("TBD");
job_ptr->wait_all_nodes = 1;
} else
set_job_alias_list(job_ptr);
} else {
/* set addrs if the job is coming from a different cluster */
set_job_node_addrs(job_ptr, job_ptr->origin_cluster);
}
}
/*
* Set addrs if:
* 1. There is an alias_list (cloud/dynamic nodes) and it isn't TBD (nodes are
* powering up).
* 2. No alias_list but job/request is from a different cluster.
*/
extern void set_job_node_addrs(job_record_t *job_ptr,
const char *origin_cluster)
{
if (!job_ptr->node_addrs &&
job_ptr->node_bitmap &&
bit_set_count(job_ptr->node_bitmap) &&
((!job_ptr->alias_list && /* remote job */
origin_cluster &&
xstrcmp(origin_cluster, slurm_conf.cluster_name)) ||
(job_ptr->alias_list && xstrcmp(job_ptr->alias_list, "TBD")))) {
node_record_t *node_ptr;
job_ptr->node_addrs =
xcalloc(bit_set_count(job_ptr->node_bitmap),
sizeof(slurm_addr_t));
for (int i = 0, addr_index = 0;
(node_ptr = next_node_bitmap(job_ptr->node_bitmap,
&i));
i++) {
slurm_conf_get_addr(node_ptr->name,
&job_ptr->node_addrs[addr_index++],
0);
}
}
}
/* Set a job's alias_list string */
extern void set_job_alias_list(job_record_t *job_ptr)
{
node_record_t *node_ptr;
xfree(job_ptr->alias_list);
if (cloud_dns && bit_super_set(job_ptr->node_bitmap, cloud_node_bitmap))
return;
for (int i = 0; (node_ptr = next_node_bitmap(job_ptr->node_bitmap, &i));
i++) {
if (IS_NODE_DYNAMIC_FUTURE(node_ptr) ||
IS_NODE_DYNAMIC_NORM(node_ptr) ||
(!cloud_dns && IS_NODE_CLOUD(node_ptr))) {
if (job_ptr->alias_list)
xstrcat(job_ptr->alias_list, ",");
xstrfmtcat(job_ptr->alias_list, "%s:[%s]:%s",
node_ptr->name, node_ptr->comm_name,
node_ptr->node_hostname);
}
}
set_job_node_addrs(job_ptr, job_ptr->origin_cluster);
}
extern void set_job_features_use(job_details_t *details_ptr)
{
if (!details_ptr)
return;
if (details_ptr->prefer) {
details_ptr->features_use = details_ptr->prefer;
details_ptr->feature_list_use = details_ptr->prefer_list;
} else {
details_ptr->features_use = details_ptr->features;
details_ptr->feature_list_use = details_ptr->feature_list;
}
}
/*
* deallocate_nodes - for a given job, deallocate its nodes and make
* their state NODE_STATE_COMPLETING also release the job's licenses
* and resources reserved by accounting policy association
* IN job_ptr - pointer to terminating job (already in some COMPLETING state)
* IN timeout - true if job exhausted time limit, send REQUEST_KILL_TIMELIMIT
* RPC instead of REQUEST_TERMINATE_JOB
* IN suspended - true if job was already suspended (node's run_job_cnt
* already decremented);
* IN preempted - true if job is being preempted
*/
extern void deallocate_nodes(job_record_t *job_ptr, bool timeout,
bool suspended, bool preempted)
{
kill_job_msg_t *kill_job = NULL;
agent_arg_t *agent_args = NULL;
node_record_t *node_ptr;
hostlist_t *hostlist = NULL;
uint16_t use_protocol_version = 0;
uint16_t msg_flags = 0;
xassert(job_ptr);
xassert(job_ptr->details);
log_flag(TRACE_JOBS, "%s: %pJ", __func__, job_ptr);
acct_policy_job_fini(job_ptr, false);
node_mgr_make_node_blocked(job_ptr, false);
if (select_g_job_fini(job_ptr) != SLURM_SUCCESS)
error("select_g_job_fini(%pJ): %m", job_ptr);
/* Release any job-related switch data */
switch_g_job_complete(job_ptr);
epilog_slurmctld(job_ptr);
if (!job_ptr->details->prolog_running)
hostlist = hostlist_create(NULL);
if (!job_ptr->node_bitmap_cg)
build_cg_bitmap(job_ptr);
use_protocol_version = SLURM_PROTOCOL_VERSION;
for (int i = 0;
(node_ptr = next_node_bitmap(job_ptr->node_bitmap_cg, &i)); i++) {
/* Sync up conditionals with make_node_comp() */
if (IS_NODE_DOWN(node_ptr) ||
IS_NODE_POWERED_DOWN(node_ptr) ||
IS_NODE_POWERING_UP(node_ptr)) {
/* Issue the KILL RPC, but don't verify response */
bit_clear(job_ptr->node_bitmap_cg, i);
job_update_tres_cnt(job_ptr, i);
/*
* node_cnt indicates how many nodes we are waiting
* to get epilog complete messages from, so do not
* count down nodes. NOTE: The job's node_cnt will not
* match the number of entries in the node string
* during its completion.
*/
job_ptr->node_cnt--;
}
make_node_comp(node_ptr, job_ptr, suspended);
if (hostlist &&
!IS_NODE_POWERED_DOWN(node_ptr) &&
!IS_NODE_POWERING_UP(node_ptr)) {
hostlist_push_host(hostlist, node_ptr->name);
if (use_protocol_version > node_ptr->protocol_version) {
use_protocol_version =
node_ptr->protocol_version;
debug3("%s: protocol version downgraded to %u from node %s",
__func__, use_protocol_version,
node_ptr->name);
}
if (PACK_FANOUT_ADDRS(node_ptr))
msg_flags |= SLURM_PACK_ADDRS;
}
}
if (job_ptr->details->prolog_running) {
/*
* Job was configuring when it was cancelled and epilog wasn't
* run on the nodes, so cleanup the nodes now. Final cleanup
* will happen after EpilogSlurmctld is done.
*/
if (job_ptr->node_bitmap_cg) {
/*
* Call cleanup_completing before job_epilog_complete or
* we will end up requeuing there before this is called.
*/
cleanup_completing(job_ptr, false);
/*
* job_epilog_complete() can free
* job_ptr->node_bitmap_cg
*/
for (int i = 0;
job_ptr->node_bitmap_cg &&
(node_ptr = next_node_bitmap(
job_ptr->node_bitmap_cg, &i));
i++) {
job_epilog_complete(job_ptr->job_id,
node_ptr->name, 0);
}
}
return;
}
/* Can not wait for epilog complete to release licenses and
* update gang scheduling table */
cleanup_completing(job_ptr, false);
resv_replace_update(job_ptr);
if (!hostlist || !hostlist_count(hostlist)) {
hostlist_destroy(hostlist);
return;
}
if (job_ptr->bit_flags & EXTERNAL_JOB) {
debug("%s: %pJ is external, no need to wait to complete",
__func__, job_ptr);
for (int i = 0;
(node_ptr = next_node_bitmap(job_ptr->node_bitmap_cg, &i));
i++) {
make_node_idle(node_ptr, job_ptr);
}
hostlist_destroy(hostlist);
return;
}
agent_args = xmalloc(sizeof(agent_arg_t));
if (timeout)
agent_args->msg_type = REQUEST_KILL_TIMELIMIT;
else if (preempted)
agent_args->msg_type = REQUEST_KILL_PREEMPTED;
else
agent_args->msg_type = REQUEST_TERMINATE_JOB;
agent_args->retry = 0; /* re_kill_job() resends as needed */
agent_args->protocol_version = use_protocol_version;
agent_args->hostlist = hostlist;
agent_args->node_count = hostlist_count(hostlist);
agent_args->msg_flags = msg_flags;
last_node_update = time(NULL);
kill_job = create_kill_job_msg(job_ptr, use_protocol_version);
kill_job->nodes = xstrdup(job_ptr->nodes);
agent_args->msg_args = kill_job;
set_agent_arg_r_uid(agent_args, SLURM_AUTH_UID_ANY);
agent_queue_request(agent_args);
}
static void _log_feature_nodes(job_feature_t *job_feat_ptr)
{
char *tmp1, *tmp2, *tmp3, *tmp4 = NULL;
if (!(slurm_conf.debug_flags & DEBUG_FLAG_NODE_FEATURES))
return;
if (job_feat_ptr->op_code == FEATURE_OP_OR)
tmp3 = "OR";
else if (job_feat_ptr->op_code == FEATURE_OP_AND)
tmp3 = "AND";
else if (job_feat_ptr->op_code == FEATURE_OP_MOR)
tmp3 = "MOR";
else if (job_feat_ptr->op_code == FEATURE_OP_XAND)
tmp3 = "XAND";
else if (job_feat_ptr->op_code == FEATURE_OP_END)
tmp3 = "END";
else {
xstrfmtcat(tmp4, "UNKNOWN:%u", job_feat_ptr->op_code);
tmp3 = tmp4;
}
tmp1 = bitmap2node_name(job_feat_ptr->node_bitmap_active);
tmp2 = bitmap2node_name(job_feat_ptr->node_bitmap_avail);
log_flag(NODE_FEATURES, "%s: FEAT:%s COUNT:%u BRACKET:%u PAREN:%d OP:%s ACTIVE:%s AVAIL:%s",
__func__, job_feat_ptr->name, job_feat_ptr->count,
job_feat_ptr->bracket, job_feat_ptr->paren, tmp3, tmp1, tmp2);
xfree(tmp1);
xfree(tmp2);
xfree(tmp4);
}
/*
* For every element in the feature_list, identify the nodes with that feature
* either active or available and set the feature_list's node_bitmap_active and
* node_bitmap_avail fields accordingly.
*/
extern void find_feature_nodes(list_t *feature_list, bool can_reboot)
{
list_itr_t *feat_iter;
job_feature_t *job_feat_ptr;
node_feature_t *node_feat_ptr;
if (!feature_list)
return;
feat_iter = list_iterator_create(feature_list);
while ((job_feat_ptr = list_next(feat_iter))) {
FREE_NULL_BITMAP(job_feat_ptr->node_bitmap_active);
FREE_NULL_BITMAP(job_feat_ptr->node_bitmap_avail);
node_feat_ptr = list_find_first(active_feature_list,
list_find_feature,
job_feat_ptr->name);
if (node_feat_ptr && node_feat_ptr->node_bitmap) {
job_feat_ptr->node_bitmap_active =
bit_copy(node_feat_ptr->node_bitmap);
} else { /* This feature not active */
job_feat_ptr->node_bitmap_active =
bit_alloc(node_record_count);
}
if (can_reboot && job_feat_ptr->changeable) {
node_feat_ptr = list_find_first(avail_feature_list,
list_find_feature,
job_feat_ptr->name);
if (node_feat_ptr && node_feat_ptr->node_bitmap) {
job_feat_ptr->node_bitmap_avail =
bit_copy(node_feat_ptr->node_bitmap);
} else { /* This feature not available */
job_feat_ptr->node_bitmap_avail =
bit_alloc(node_record_count);
}
} else if (job_feat_ptr->node_bitmap_active) {
job_feat_ptr->node_bitmap_avail =
bit_copy(job_feat_ptr->node_bitmap_active);
}
_log_feature_nodes(job_feat_ptr);
}
list_iterator_destroy(feat_iter);
}
/*
* _match_feature - determine which of the job features are now inactive
* IN feature_list - Job's feature request list
* OUT inactive_bitmap - Nodes with this as inactive feature
* RET 1 if some nodes with this inactive feature, 0 no inactive feature
* NOTE: Currently fully supports only AND/OR of features, not XAND/MOR
*/
static int _match_feature(list_t *feature_list, bitstr_t **inactive_bitmap)
{
list_itr_t *job_feat_iter;
job_feature_t *job_feat_ptr;
int last_op = FEATURE_OP_AND, last_paren_op = FEATURE_OP_AND;
int i, last_paren_cnt = 0;
bitstr_t *feature_bitmap, *paren_bitmap = NULL, *work_bitmap;
xassert(inactive_bitmap);
if (!feature_list || /* nothing to look for */
(node_features_g_count() == 0)) /* No inactive features */
return 0;
feature_bitmap = node_conf_get_active_bitmap();
work_bitmap = feature_bitmap;
job_feat_iter = list_iterator_create(feature_list);
while ((job_feat_ptr = list_next(job_feat_iter))) {
if (last_paren_cnt < job_feat_ptr->paren) {
/* Start of expression in parenthesis */
last_paren_op = last_op;
last_op = FEATURE_OP_AND;
FREE_NULL_BITMAP(paren_bitmap);
paren_bitmap = node_conf_get_active_bitmap();
work_bitmap = paren_bitmap;
}
if (job_feat_ptr->node_bitmap_avail) {
if (last_op == FEATURE_OP_AND) {
bit_and(work_bitmap,
job_feat_ptr->node_bitmap_active);
} else if (last_op == FEATURE_OP_OR) {
bit_or(work_bitmap,
job_feat_ptr->node_bitmap_active);
} else { /* FEATURE_OP_MOR or FEATURE_OP_XAND */
bit_and(work_bitmap,
job_feat_ptr->node_bitmap_active);
}
} else { /* feature not found */
if (last_op == FEATURE_OP_AND) {
bit_clear_all(work_bitmap);
}
}
if (last_paren_cnt > job_feat_ptr->paren) {
/* End of expression in parenthesis */
if (last_paren_op == FEATURE_OP_AND) {
bit_and(feature_bitmap, work_bitmap);
} else if (last_paren_op == FEATURE_OP_OR) {
bit_or(feature_bitmap, work_bitmap);
} else { /* FEATURE_OP_MOR or FEATURE_OP_XAND */
bit_and(feature_bitmap, work_bitmap);
}
work_bitmap = feature_bitmap;
}
last_op = job_feat_ptr->op_code;
last_paren_cnt = job_feat_ptr->paren;
}
list_iterator_destroy(job_feat_iter);
#if 0
{
char tmp[32];
bit_fmt(tmp, sizeof(tmp), work_bitmap);
info("%s: NODE_BITMAP:%s", __func__, tmp);
}
#endif
FREE_NULL_BITMAP(paren_bitmap);
i = bit_ffc(feature_bitmap);
if (i == -1) { /* No required node features inactive */
FREE_NULL_BITMAP(feature_bitmap);
return 0;
}
bit_not(feature_bitmap);
*inactive_bitmap = feature_bitmap;
return 1;
}
/*
* For a given job, if the available nodes differ from those with currently
* active features, return a bitmap of nodes with the job's required
* features currently active
* IN job_ptr - job requesting resource allocation
* IN avail_bitmap - nodes currently available for this job
* OUT active_bitmap - nodes with job's features currently active, NULL if
* identical to avail_bitmap
* NOTE: Currently fully supports only AND/OR of features, not XAND/MOR
*/
extern void build_active_feature_bitmap(job_record_t *job_ptr,
bitstr_t *avail_bitmap,
bitstr_t **active_bitmap)
{
job_details_t *details_ptr = job_ptr->details;
bitstr_t *tmp_bitmap = NULL;
bool can_reboot;
*active_bitmap = NULL;
if (!details_ptr->feature_list_use || /* nothing to look for */
(node_features_g_count() == 0)) /* No inactive features */
return;
can_reboot = node_features_g_user_update(job_ptr->user_id);
find_feature_nodes(details_ptr->feature_list_use, can_reboot);
if (_match_feature(details_ptr->feature_list_use, &tmp_bitmap) == 0)
return; /* No inactive features */
bit_not(tmp_bitmap);
if (bit_super_set(avail_bitmap, tmp_bitmap)) {
FREE_NULL_BITMAP(tmp_bitmap);
return;
}
bit_and(tmp_bitmap, avail_bitmap);
*active_bitmap = tmp_bitmap;
}
/* Return bitmap of nodes with all specified features currently active */
extern bitstr_t *build_active_feature_bitmap2(char *reboot_features)
{
const char *delim = ",";
char *tmp, *tok, *save_ptr = NULL;
bitstr_t *active_node_bitmap = NULL;
node_feature_t *node_feat_ptr;
if (!reboot_features || (reboot_features[0] == '\0')) {
active_node_bitmap = node_conf_get_active_bitmap();
return active_node_bitmap;
}
tmp = xstrdup(reboot_features);
tok = strtok_r(tmp, delim, &save_ptr);
while (tok) {
node_feat_ptr = list_find_first(active_feature_list,
list_find_feature, tok);
if (node_feat_ptr && node_feat_ptr->node_bitmap) {
/*
* Found feature, add nodes with this feature and
* remove nodes without this feature (bit_and)
*/
if (!active_node_bitmap)
active_node_bitmap =
bit_copy(node_feat_ptr->node_bitmap);
else
bit_and(active_node_bitmap,
node_feat_ptr->node_bitmap);
} else {
/*
* Feature not found in any nodes, so we definitely
* need to reboot all of the nodes
*/
if (!active_node_bitmap)
active_node_bitmap =
bit_alloc(node_record_count);
else
bit_clear_all(active_node_bitmap);
break;
}
tok = strtok_r(NULL, delim, &save_ptr);
}
xfree(tmp);
return active_node_bitmap;
}
/*
* Decide if a job can share nodes with other jobs based on the
* following three input parameters:
*
* IN user_flag - may be 0 (do not share nodes), 1 (node sharing allowed),
* or any other number means "don't care"
* IN part_max_share - current partition's node sharing policy
*
*
* The followed table details the node SHARED state for the various scenarios
*
* part= part= part= part=
* cons_tres user_request EXCLUS NO YES FORCE
* -------- ------------ ------ ----- ----- -----
* no default whole whole whole whole/O
* no exclusive whole whole whole whole/O
* no share=yes whole whole whole/O whole/O
* yes default whole share share share/O
* yes exclusive whole whole whole whole/O
* yes share=yes whole share share/O share/O
*
* whole = entire node is allocated to the job
* share = less than entire node may be allocated to the job
* -/O = resources can be over-committed (e.g. gang scheduled)
*
* part->max_share:
* &SHARED_FORCE = FORCE
* 0 = EXCLUSIVE
* 1 = NO
* > 1 = YES
*
* job_ptr->details->share_res:
* 0 = default or share=no
* 1 = share=yes
*
* job_ptr->details->whole_node:
* 0 = default
* WHOLE_NODE_REQUIRED = 1 = exclusive
* WHOLE_NODE_USER = 2 = user
* WHOLE_NODE_MCS = 3 = mcs
*
* Return values:
* 0 = requires idle nodes
* 1 = can use non-idle nodes
*/
static int _resolve_shared_status(job_record_t *job_ptr,
uint16_t part_max_share)
{
if (job_ptr->reboot)
return 0;
/* no sharing if partition OverSubscribe=EXCLUSIVE */
if (part_max_share == 0) {
job_ptr->details->whole_node |= WHOLE_NODE_REQUIRED;
job_ptr->details->share_res = 0;
return 0;
}
/* sharing if partition OverSubscribe=FORCE with count > 1 */
if ((part_max_share & SHARED_FORCE) &&
((part_max_share & (~SHARED_FORCE)) > 1)) {
job_ptr->details->share_res = 1;
return 1;
}
if (running_cons_tres()) {
if ((job_ptr->details->share_res == 0) ||
(job_ptr->details->whole_node & WHOLE_NODE_REQUIRED)) {
job_ptr->details->share_res = 0;
return 0;
}
return 1;
} else {
job_ptr->details->whole_node |= WHOLE_NODE_REQUIRED;
if (part_max_share == 1) { /* partition is OverSubscribe=NO */
job_ptr->details->share_res = 0;
return 0;
}
/* share if the user requested it */
if (job_ptr->details->share_res == 1)
return 1;
job_ptr->details->share_res = 0;
return 0;
}
}
typedef struct {
job_record_t *job_ptr;
bitstr_t *usable_node_mask;
} foreach_filter_by_node_t;
static int _foreach_filter_by_node_owner(void *x, void *arg)
{
job_record_t *job_ptr2 = x;
foreach_filter_by_node_t *argstruct = arg;
job_record_t *job_ptr = argstruct->job_ptr;
bitstr_t *usable_node_mask = argstruct->usable_node_mask;
if (IS_JOB_PENDING(job_ptr2) || IS_JOB_COMPLETED(job_ptr2) ||
(job_ptr->user_id == job_ptr2->user_id) || !job_ptr2->node_bitmap)
return 0;
bit_and_not(usable_node_mask, job_ptr2->node_bitmap);
return 0;
}
/*
* Remove nodes from consideration for allocation based upon "ownership" by
* other users
* job_ptr IN - Job to be scheduled
* usable_node_mask IN/OUT - Nodes available for use by this job's user
*/
extern void filter_by_node_owner(job_record_t *job_ptr,
bitstr_t *usable_node_mask)
{
node_record_t *node_ptr;
int i;
foreach_filter_by_node_t argstruct = { .job_ptr = job_ptr,
.usable_node_mask =
usable_node_mask };
if ((job_ptr->details->whole_node & WHOLE_NODE_USER) ||
(job_ptr->part_ptr->flags & PART_FLAG_EXCLUSIVE_USER)) {
/* Need to remove all nodes allocated to any active job from
* any other user */
list_for_each(job_list, _foreach_filter_by_node_owner,
&argstruct);
return;
}
/* Need to filter out any nodes exclusively allocated to other users */
for (i = 0; (node_ptr = next_node(&i)); i++) {
if ((node_ptr->owner != NO_VAL) &&
(node_ptr->owner != job_ptr->user_id))
bit_clear(usable_node_mask, node_ptr->index);
}
}
/*
* Remove nodes from consideration for allocation based upon "mcs" by
* other users
* job_ptr IN - Job to be scheduled
* usable_node_mask IN/OUT - Nodes available for use by this job's mcs
*/
extern void filter_by_node_mcs(job_record_t *job_ptr, int mcs_select,
bitstr_t *usable_node_mask)
{
node_record_t *node_ptr;
int i;
/* Need to filter out any nodes allocated with other mcs */
if (job_ptr->mcs_label && (mcs_select == 1)) {
for (i = 0; (node_ptr = next_node(&i)); i++) {
/* if there is a mcs_label -> OK if it's the same */
if ((node_ptr->mcs_label != NULL) &&
xstrcmp(node_ptr->mcs_label,job_ptr->mcs_label)) {
bit_clear(usable_node_mask, node_ptr->index);
}
/* if no mcs_label -> OK if no jobs running */
if ((node_ptr->mcs_label == NULL) &&
(node_ptr->run_job_cnt != 0)) {
bit_clear(usable_node_mask, node_ptr->index);
}
}
} else {
for (i = 0; (node_ptr = next_node(&i)); i++) {
if (node_ptr->mcs_label != NULL) {
bit_clear(usable_node_mask, node_ptr->index);
}
}
}
}
/*
* Remove nodes from the "avail_node_bitmap" which need to be rebooted in order
* to be used if the job's "delay_boot" time has not yet been reached.
*/
static void _filter_by_node_feature(job_record_t *job_ptr,
struct node_set *node_set_ptr,
int node_set_size)
{
int i;
if ((job_ptr->details == NULL) ||
((job_ptr->details->begin_time != 0) &&
((job_ptr->details->begin_time + job_ptr->delay_boot) <=
time(NULL))))
return;
for (i = 0; i < node_set_size; i++) {
if (node_set_ptr[i].flags & NODE_SET_REBOOT) {
bit_and_not(avail_node_bitmap,
node_set_ptr[i].my_bitmap);
}
}
}
static void _find_qos_grp_node_bitmap(job_record_t *job_ptr,
slurmdb_qos_rec_t *qos_ptr,
bitstr_t **grp_node_bitmap,
bool *per_grp_limit,
bool *per_user_limit,
bool *per_acct_limit)
{
slurmdb_used_limits_t *used_limits = NULL;
if (!qos_ptr || !qos_ptr->usage)
return;
if (!*per_grp_limit &&
qos_ptr->usage->grp_node_bitmap &&
(qos_ptr->grp_tres_ctld[TRES_ARRAY_NODE] != INFINITE64)) {
*per_grp_limit = true;
*grp_node_bitmap = bit_copy(qos_ptr->usage->grp_node_bitmap);
}
if (!*per_user_limit &&
(qos_ptr->max_tres_pu_ctld[TRES_ARRAY_NODE] != INFINITE64)) {
*per_user_limit = true;
used_limits = acct_policy_get_user_used_limits(
&qos_ptr->usage->user_limit_list,
job_ptr->user_id);
if (used_limits && used_limits->node_bitmap) {
if (*grp_node_bitmap)
bit_or(*grp_node_bitmap,
used_limits->node_bitmap);
else
*grp_node_bitmap =
bit_copy(used_limits->node_bitmap);
}
}
if (!*per_acct_limit &&
job_ptr->assoc_ptr &&
(qos_ptr->max_tres_pa_ctld[TRES_ARRAY_NODE] != INFINITE64)) {
*per_acct_limit = true;
used_limits = acct_policy_get_acct_used_limits(
&qos_ptr->usage->acct_limit_list,
job_ptr->assoc_ptr->acct);
if (used_limits && used_limits->node_bitmap) {
if (*grp_node_bitmap)
bit_or(*grp_node_bitmap,
used_limits->node_bitmap);
else
*grp_node_bitmap =
bit_copy(used_limits->node_bitmap);
}
}
}
/*
* For a given job, return a bitmap of nodes to be preferred in it's allocation
*/
static bitstr_t *_find_grp_node_bitmap(job_record_t *job_ptr)
{
bitstr_t *grp_node_bitmap = NULL;
slurmdb_qos_rec_t *qos_ptr1 = NULL, *qos_ptr2 = NULL;
bool per_acct_limit = false, per_user_limit = false,
per_grp_limit = false;
assoc_mgr_lock_t qos_read_locks =
{ .assoc = READ_LOCK, .qos = READ_LOCK };
slurmdb_assoc_rec_t *assoc_ptr = job_ptr->assoc_ptr;
/* check to see if we are enforcing associations */
if (!(accounting_enforce & ACCOUNTING_ENFORCE_LIMITS))
return NULL;
assoc_mgr_lock(&qos_read_locks);
acct_policy_set_qos_order(job_ptr, &qos_ptr1, &qos_ptr2);
_find_qos_grp_node_bitmap(job_ptr, qos_ptr1, &grp_node_bitmap,
&per_grp_limit,
&per_user_limit,
&per_acct_limit);
_find_qos_grp_node_bitmap(job_ptr, qos_ptr2, &grp_node_bitmap,
&per_grp_limit,
&per_user_limit,
&per_acct_limit);
while (assoc_ptr && assoc_ptr->usage && !per_grp_limit) {
if (assoc_ptr->usage->grp_node_bitmap &&
(assoc_ptr->grp_tres_ctld[TRES_ARRAY_NODE] != INFINITE64)) {
per_grp_limit = true;
if (grp_node_bitmap)
bit_or(grp_node_bitmap,
assoc_ptr->usage->grp_node_bitmap);
else
grp_node_bitmap = bit_copy(assoc_ptr->usage->
grp_node_bitmap);
break;
}
assoc_ptr = assoc_ptr->usage->parent_assoc_ptr;
}
assoc_mgr_unlock(&qos_read_locks);
return grp_node_bitmap;
}
/*
* If the job has required feature counts, then accumulate those
* required resources using multiple calls to _pick_best_nodes()
* and adding those selected nodes to the job's required node list.
* Upon completion, return job's requirements to match the values
* which were in effect upon calling this function.
* Input and output are the same as _pick_best_nodes().
*/
static int _get_req_features(struct node_set *node_set_ptr, int node_set_size,
bitstr_t **select_bitmap, job_record_t *job_ptr,
part_record_t *part_ptr, uint32_t min_nodes,
uint32_t max_nodes, uint32_t req_nodes,
bool test_only, list_t **preemptee_job_list,
bool can_reboot, bool submission)
{
uint32_t saved_min_nodes, saved_job_min_nodes, saved_job_num_tasks;
bitstr_t *saved_req_node_bitmap = NULL;
bitstr_t *inactive_bitmap = NULL;
uint32_t saved_min_cpus, saved_req_nodes;
int resv_rc = SLURM_SUCCESS, tmp_node_set_size;
int mcs_select = 0;
struct node_set *tmp_node_set_ptr, *prev_node_set_ptr;
int error_code = SLURM_SUCCESS, i;
bitstr_t *feature_bitmap, *accumulate_bitmap = NULL;
bitstr_t *save_avail_node_bitmap = NULL, *resv_bitmap = NULL;
bitstr_t *save_share_node_bitmap = NULL;
list_t *preemptee_candidates = NULL;
bool old_feat_change = false;
bool has_xand = false;
bool resv_overlap = false;
resv_exc_t resv_exc = { 0 };
/*
* Mark nodes reserved for other jobs as off limit for this job.
* If the job has a reservation, we've already limited the contents
* of select_bitmap to those nodes. Assume node reboot required
* since we have not selected the compute nodes yet.
*/
if (job_ptr->resv_name == NULL) {
time_t start_res = time(NULL);
resv_rc = job_test_resv(job_ptr, &start_res, false,
&resv_bitmap, &resv_exc,
&resv_overlap, true);
if ((resv_rc == ESLURM_NODES_BUSY) ||
(resv_rc == ESLURM_RESERVATION_MAINT)) {
save_avail_node_bitmap = avail_node_bitmap;
avail_node_bitmap = bit_alloc(node_record_count);
FREE_NULL_BITMAP(resv_bitmap);
/*
* Continue executing through _pick_best_nodes() below
* in order reject job if it can never run
*/
} else if (resv_rc != SLURM_SUCCESS) {
FREE_NULL_BITMAP(resv_bitmap);
reservation_delete_resv_exc_parts(&resv_exc);
return ESLURM_NODES_BUSY; /* reserved */
} else if (resv_bitmap &&
(!bit_equal(resv_bitmap, avail_node_bitmap))) {
bit_and(resv_bitmap, avail_node_bitmap);
save_avail_node_bitmap = avail_node_bitmap;
if (slurm_conf.debug_flags & DEBUG_FLAG_RESERVATION &&
!bit_equal(avail_node_bitmap, resv_bitmap)) {
bitstr_t *removed_nodes =
bit_copy(save_avail_node_bitmap);
bit_and_not(removed_nodes, resv_bitmap);
log_flag(RESERVATION, "Advanced reservation removed nodes:%s from consideration for %pJ",
bitmap2node_name(removed_nodes),
job_ptr);
FREE_NULL_BITMAP(removed_nodes);
}
avail_node_bitmap = resv_bitmap;
resv_bitmap = NULL;
} else {
FREE_NULL_BITMAP(resv_bitmap);
}
} else {
time_t start_res = time(NULL);
/*
* We do not care about return value.
* We are just interested in resv_exc being filled in
*/
(void) job_test_resv(job_ptr, &start_res, false, &resv_bitmap,
&resv_exc, &resv_overlap, true);
FREE_NULL_BITMAP(resv_bitmap);
}
if (submission)
resv_overlap = false;
if (!save_avail_node_bitmap)
save_avail_node_bitmap = bit_copy(avail_node_bitmap);
save_share_node_bitmap = bit_copy(share_node_bitmap);
filter_by_node_owner(job_ptr, share_node_bitmap);
if (can_reboot && !test_only)
_filter_by_node_feature(job_ptr, node_set_ptr, node_set_size);
if (!test_only) {
mcs_select = slurm_mcs_get_select(job_ptr);
filter_by_node_mcs(job_ptr, mcs_select, share_node_bitmap);
}
if (!test_only) {
hres_filter(job_ptr, avail_node_bitmap);
}
/* save job and request state */
saved_min_nodes = min_nodes;
saved_req_nodes = req_nodes;
saved_job_min_nodes = job_ptr->details->min_nodes;
if (job_ptr->details->req_node_bitmap) {
accumulate_bitmap = job_ptr->details->req_node_bitmap;
saved_req_node_bitmap = bit_copy(accumulate_bitmap);
job_ptr->details->req_node_bitmap = NULL;
}
saved_min_cpus = job_ptr->details->min_cpus;
/*
* Don't mess with max_cpus here since it is only set to be a limit
* and not user configurable.
*/
job_ptr->details->min_cpus = 1;
tmp_node_set_ptr = xcalloc((node_set_size * 2), sizeof(struct node_set));
/* Accumulate nodes with required feature counts. */
preemptee_candidates = slurm_find_preemptable_jobs(job_ptr);
if (job_ptr->details->feature_list_use) {
list_itr_t *feat_iter;
job_feature_t *feat_ptr;
int last_paren_cnt = 0, last_paren_opt = FEATURE_OP_AND;
bitstr_t *paren_bitmap = NULL, *work_bitmap;
uint64_t smallest_min_mem = INFINITE64;
uint64_t orig_req_mem = job_ptr->details->pn_min_memory;
bool feat_change = false;
feat_iter = list_iterator_create(
job_ptr->details->feature_list_use);
while ((feat_ptr = list_next(feat_iter))) {
bool sort_again = false;
if (last_paren_cnt < feat_ptr->paren) {
/* Start of expression in parenthesis */
if (paren_bitmap) {
error("%s@%d: %pJ has bad feature expression: %s",
__func__, __LINE__, job_ptr,
job_ptr->details->features_use);
FREE_NULL_BITMAP(paren_bitmap);
}
feat_change |= feat_ptr->changeable;
paren_bitmap =
bit_copy(feat_ptr->node_bitmap_avail);
last_paren_opt = feat_ptr->op_code;
last_paren_cnt = feat_ptr->paren;
continue;
} else if (last_paren_cnt > 0) {
feat_change |= feat_ptr->changeable;
if (last_paren_opt == FEATURE_OP_AND) {
bit_and(paren_bitmap,
feat_ptr->node_bitmap_avail);
} else {
bit_or(paren_bitmap,
feat_ptr->node_bitmap_avail);
}
last_paren_opt = feat_ptr->op_code;
last_paren_cnt = feat_ptr->paren;
if (last_paren_cnt)
continue;
work_bitmap = paren_bitmap;
} else {
/* Outside of parenthesis */
feat_change = feat_ptr->changeable;
work_bitmap = feat_ptr->node_bitmap_avail;
}
if (feat_ptr->count == 0) {
FREE_NULL_BITMAP(paren_bitmap);
continue;
}
tmp_node_set_size = 0;
/*
* _pick_best_nodes() is destructive of the node_set
* data structure, so we need to make a copy and then
* purge it
*/
for (i = 0; i < node_set_size; i++) {
if (!bit_overlap_any(node_set_ptr[i].my_bitmap,
work_bitmap))
continue;
tmp_node_set_ptr[tmp_node_set_size].
cpus_per_node =
node_set_ptr[i].cpus_per_node;
tmp_node_set_ptr[tmp_node_set_size].
real_memory =
node_set_ptr[i].real_memory;
tmp_node_set_ptr[tmp_node_set_size].node_weight =
node_set_ptr[i].node_weight;
tmp_node_set_ptr[tmp_node_set_size].sched_weight =
node_set_ptr[i].sched_weight;
tmp_node_set_ptr[tmp_node_set_size].flags =
node_set_ptr[i].flags;
tmp_node_set_ptr[tmp_node_set_size].features =
xstrdup(node_set_ptr[i].features);
tmp_node_set_ptr[tmp_node_set_size].
feature_bits =
bit_copy(node_set_ptr[i].feature_bits);
tmp_node_set_ptr[tmp_node_set_size].my_bitmap =
bit_copy(node_set_ptr[i].my_bitmap);
bit_and(tmp_node_set_ptr[tmp_node_set_size].
my_bitmap, work_bitmap);
if (accumulate_bitmap && has_xand) {
bit_and_not(tmp_node_set_ptr[
tmp_node_set_size].my_bitmap,
accumulate_bitmap);
}
tmp_node_set_ptr[tmp_node_set_size].node_cnt =
bit_set_count(tmp_node_set_ptr
[tmp_node_set_size].my_bitmap);
prev_node_set_ptr = tmp_node_set_ptr +
tmp_node_set_size;
tmp_node_set_size++;
if (test_only || !can_reboot ||
(prev_node_set_ptr->flags &
NODE_SET_REBOOT))
continue;
inactive_bitmap =
bit_copy(node_set_ptr[i].my_bitmap);
bit_and_not(inactive_bitmap,
feat_ptr->node_bitmap_active);
if (bit_ffs(inactive_bitmap) == -1) {
/* No inactive nodes (require reboot) */
FREE_NULL_BITMAP(inactive_bitmap);
continue;
}
sort_again = true;
if (bit_equal(prev_node_set_ptr->my_bitmap,
inactive_bitmap)) {
prev_node_set_ptr->flags |=
NODE_SET_REBOOT;
FREE_NULL_BITMAP(inactive_bitmap);
continue;
}
tmp_node_set_ptr[tmp_node_set_size].
cpus_per_node =
node_set_ptr[i].cpus_per_node;
tmp_node_set_ptr[tmp_node_set_size].
real_memory =
node_set_ptr[i].real_memory;
tmp_node_set_ptr[tmp_node_set_size].flags |=
NODE_SET_REBOOT;
tmp_node_set_ptr[tmp_node_set_size].features =
xstrdup(node_set_ptr[i].features);
tmp_node_set_ptr[tmp_node_set_size].
feature_bits =
bit_copy(node_set_ptr[i].feature_bits);
tmp_node_set_ptr[tmp_node_set_size].my_bitmap =
bit_copy(tmp_node_set_ptr
[tmp_node_set_size-1].my_bitmap);
bit_and(tmp_node_set_ptr[tmp_node_set_size].
my_bitmap, inactive_bitmap);
tmp_node_set_ptr[tmp_node_set_size].node_cnt =
bit_set_count(tmp_node_set_ptr
[tmp_node_set_size].my_bitmap);
bit_and_not(tmp_node_set_ptr[tmp_node_set_size-1].
my_bitmap, inactive_bitmap);
tmp_node_set_ptr[tmp_node_set_size-1].node_cnt =
bit_set_count(tmp_node_set_ptr
[tmp_node_set_size-1].my_bitmap);
tmp_node_set_size++;
FREE_NULL_BITMAP(inactive_bitmap);
}
FREE_NULL_BITMAP(paren_bitmap);
feature_bitmap = NULL;
min_nodes = feat_ptr->count;
req_nodes = feat_ptr->count;
saved_job_num_tasks = job_ptr->details->num_tasks;
job_ptr->details->min_nodes = feat_ptr->count;
job_ptr->details->min_cpus = feat_ptr->count;
/*
* Ensure that num_tasks is accurate if ntasks_per_node
* is set
*/
if (job_ptr->details->ntasks_per_node)
job_ptr->details->num_tasks = min_nodes *
job_ptr->details->ntasks_per_node;
FREE_NULL_LIST(*preemptee_job_list);
job_ptr->details->pn_min_memory = orig_req_mem;
if (sort_again) {
for (i = 0; i < tmp_node_set_size; i++)
_set_sched_weight(tmp_node_set_ptr + i);
qsort(tmp_node_set_ptr, tmp_node_set_size,
sizeof(struct node_set), _sort_node_set);
}
error_code = _pick_best_nodes(tmp_node_set_ptr,
tmp_node_set_size, &feature_bitmap,
job_ptr, part_ptr, min_nodes,
max_nodes, req_nodes, test_only,
preemptee_candidates,
preemptee_job_list, false,
&resv_exc, resv_overlap);
job_ptr->details->num_tasks = saved_job_num_tasks;
if (job_ptr->details->pn_min_memory) {
if (job_ptr->details->pn_min_memory <
smallest_min_mem)
smallest_min_mem =
job_ptr->details->pn_min_memory;
else
job_ptr->details->pn_min_memory =
smallest_min_mem;
}
#if _DEBUG
{
char *tmp_str = bitmap2node_name(feature_bitmap);
info("%pJ needs %u nodes with feature %s, using %s, error_code=%d",
job_ptr, feat_ptr->count, feat_ptr->name,
tmp_str, error_code);
xfree(tmp_str);
}
#endif
for (i = 0; i < tmp_node_set_size; i++) {
xfree(tmp_node_set_ptr[i].features);
FREE_NULL_BITMAP(tmp_node_set_ptr[i].
feature_bits);
FREE_NULL_BITMAP(tmp_node_set_ptr[i].
my_bitmap);
}
if (error_code != SLURM_SUCCESS) {
FREE_NULL_BITMAP(feature_bitmap);
break;
}
if (feature_bitmap) {
if (feat_ptr->op_code == FEATURE_OP_XAND)
has_xand = true;
if (has_xand) {
if (old_feat_change && feat_change) {
error_code =
ESLURM_MULTI_KNL_CONSTRAINT;
break;
}
old_feat_change |= feat_change;
/*
* Don't make nodes required since we
* check value on each call to
* _pick_best_nodes()
*/
} else if (job_ptr->details->req_node_bitmap) {
bit_or(job_ptr->details->
req_node_bitmap,
feature_bitmap);
} else {
job_ptr->details->req_node_bitmap =
bit_copy(feature_bitmap);
}
if (accumulate_bitmap) {
bit_or(accumulate_bitmap,
feature_bitmap);
FREE_NULL_BITMAP(feature_bitmap);
} else
accumulate_bitmap = feature_bitmap;
}
}
list_iterator_destroy(feat_iter);
if (paren_bitmap) {
error("%s@%d: %pJ has bad feature expression: %s",
__func__, __LINE__, job_ptr,
job_ptr->details->features_use);
FREE_NULL_BITMAP(paren_bitmap);
}
}
/* restore most of job state and accumulate remaining resources */
if (saved_req_node_bitmap) {
FREE_NULL_BITMAP(job_ptr->details->req_node_bitmap);
job_ptr->details->req_node_bitmap =
bit_copy(saved_req_node_bitmap);
}
if (accumulate_bitmap) {
uint32_t node_cnt;
if (job_ptr->details->req_node_bitmap) {
bit_or(job_ptr->details->req_node_bitmap,
accumulate_bitmap);
FREE_NULL_BITMAP(accumulate_bitmap);
} else
job_ptr->details->req_node_bitmap = accumulate_bitmap;
node_cnt = bit_set_count(job_ptr->details->req_node_bitmap);
job_ptr->details->min_cpus = MAX(saved_min_cpus, node_cnt);
min_nodes = MAX(saved_min_nodes, node_cnt);
job_ptr->details->min_nodes = min_nodes;
req_nodes = MAX(min_nodes, req_nodes);
if (req_nodes > max_nodes)
error_code = ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE;
} else {
min_nodes = saved_min_nodes;
req_nodes = saved_req_nodes;
job_ptr->details->min_cpus = saved_min_cpus;
job_ptr->details->min_nodes = saved_job_min_nodes;
}
#if _DEBUG
{
char *tmp_str = bitmap2node_name(job_ptr->details->req_node_bitmap);
info("%pJ requires %d:%d:%d req_nodes:%s err:%u",
job_ptr, min_nodes, req_nodes, max_nodes, tmp_str, error_code);
xfree(tmp_str);
}
#endif
xfree(tmp_node_set_ptr);
if (error_code == SLURM_SUCCESS) {
FREE_NULL_LIST(*preemptee_job_list);
error_code = _pick_best_nodes(node_set_ptr, node_set_size,
select_bitmap, job_ptr, part_ptr, min_nodes,
max_nodes, req_nodes, test_only,
preemptee_candidates, preemptee_job_list,
has_xand, &resv_exc, resv_overlap);
}
if ((resv_rc == ESLURM_RESERVATION_MAINT) &&
(error_code == ESLURM_NODE_NOT_AVAIL))
error_code = ESLURM_RESERVATION_MAINT;
#if _DEBUG
{
char *tmp_str = bitmap2node_name(*select_bitmap);
info("%pJ allocated nodes:%s err:%u", job_ptr, tmp_str, error_code);
xfree(tmp_str);
}
#endif
FREE_NULL_LIST(preemptee_candidates);
/* restore job's initial required node bitmap */
FREE_NULL_BITMAP(job_ptr->details->req_node_bitmap);
job_ptr->details->req_node_bitmap = saved_req_node_bitmap;
job_ptr->details->min_cpus = saved_min_cpus;
job_ptr->details->min_nodes = saved_job_min_nodes;
/* Restore available node bitmap, ignoring reservations */
if (save_avail_node_bitmap) {
FREE_NULL_BITMAP(avail_node_bitmap);
avail_node_bitmap = save_avail_node_bitmap;
}
if (save_share_node_bitmap) {
FREE_NULL_BITMAP(share_node_bitmap);
share_node_bitmap = save_share_node_bitmap;
}
reservation_delete_resv_exc_parts(&resv_exc);
return error_code;
}
static void _sync_node_weight(struct node_set *node_set_ptr, int node_set_size)
{
node_record_t *node_ptr;
for (int s = 0; s < node_set_size; s++) {
if (!node_set_ptr[s].my_bitmap)
continue; /* No nodes in this set */
for (int i = 0;
(node_ptr = next_node_bitmap(node_set_ptr[s].my_bitmap,
&i));
i++) {
node_ptr->sched_weight = node_set_ptr[s].sched_weight;
}
}
}
static int _bit_or_cond_internal(void *x, void *arg)
{
job_record_t *job_ptr = (job_record_t *)x;
bitstr_t *bitmap = (bitstr_t *)arg;
if (!IS_JOB_RUNNING(job_ptr) || job_ptr->details->share_res ||
!job_ptr->job_resrcs)
return 0;
bit_or(bitmap, job_ptr->job_resrcs->node_bitmap);
return 0;
}
static void _bit_or_cond(job_record_t *job_ptr, bitstr_t *bitmap)
{
if (!job_ptr->het_job_list)
_bit_or_cond_internal(job_ptr, bitmap);
else
list_for_each_nobreak(job_ptr->het_job_list,
_bit_or_cond_internal, bitmap);
}
/*
* _pick_best_nodes - from a weight order list of all nodes satisfying a
* job's specifications, select the "best" for use
* IN node_set_ptr - pointer to node specification information
* IN node_set_size - number of entries in records pointed to by node_set_ptr
* OUT select_bitmap - returns bitmap of selected nodes, must FREE_NULL_BITMAP
* IN job_ptr - pointer to job being scheduled
* IN part_ptr - pointer to the partition in which the job is being scheduled
* IN min_nodes - minimum count of nodes required by the job
* IN max_nodes - maximum count of nodes required by the job (0==no limit)
* IN req_nodes - requested (or desired) count of nodes
* IN test_only - do not actually allocate resources
* IN/OUT preemptee_job_list - list of pointers to jobs to be preempted
* IN resv_exc_ptr - Various TRES which can not be used
* NULL on first entry
* IN has_xand - set of the constraint list includes XAND operators *and*
* we have already satisfied them all
* in resv_overlap - designated reservation overlaps another reservation
* RET SLURM_SUCCESS on success,
* ESLURM_NODES_BUSY if request can not be satisfied now,
* ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE if request can never
* be satisfied,
* ESLURM_REQUESTED_PART_CONFIG_UNAVAILABLE if the job can not be
* initiated until the partition's configuration changes or
* ESLURM_NODE_NOT_AVAIL if required nodes are DOWN or DRAINED
* ESLURM_RESERVATION_BUSY if requested reservation overlaps another
* NOTE: the caller must FREE_NULL_BITMAP memory pointed to by select_bitmap
* Notes: The algorithm is
* 1) If required node list is specified, determine implicitly required
* processor and node count
* 2) Determine how many disjoint required "features" are represented
* (e.g. "FS1|FS2|FS3")
* 3) For each feature: find matching node table entries, identify nodes
* that are up and available (idle or shared) and add them to a bit
* map
* 4) Select_g_job_test() to select the "best" of those based upon
* topology and/or workload
* 5) If request can't be satisfied now, execute select_g_job_test()
* against the list of nodes that exist in any state (perhaps DOWN
* DRAINED or ALLOCATED) to determine if the request can
* ever be satisfied.
*/
static int _pick_best_nodes(struct node_set *node_set_ptr, int node_set_size,
bitstr_t **select_bitmap, job_record_t *job_ptr,
part_record_t *part_ptr, uint32_t min_nodes,
uint32_t max_nodes, uint32_t req_nodes,
bool test_only, list_t *preemptee_candidates,
list_t **preemptee_job_list, bool has_xand,
resv_exc_t *resv_exc_ptr, bool resv_overlap)
{
int error_code = SLURM_SUCCESS, i, j, pick_code = SLURM_SUCCESS;
int total_nodes = 0, avail_nodes = 0;
bitstr_t *avail_bitmap = NULL, *total_bitmap = NULL;
bitstr_t *backup_bitmap = NULL;
bitstr_t *possible_bitmap = NULL;
bitstr_t *node_set_map;
int max_feature, min_feature;
bool runable_ever = false; /* Job can ever run */
bool runable_avail = false; /* Job can run with available nodes */
bool tried_sched = false; /* Tried to schedule with avail nodes */
bool preempt_flag = false;
bool nodes_busy = false;
bool licenses_unavailable = false;
int shared = 0, select_mode;
list_t *preemptee_cand = NULL;
/*
* Since you could potentially have multiple features and the
* job might not request memory we need to keep track of a minimum
* from the selected features. This is to fulfill commit
* 700e7b1d4e9.
* If no memory is requested but we are running with
* SELECT_*_MEMORY and the request is for
* nodes of different memory sizes we need to reset the
* pn_min_memory as select_g_job_test can
* alter that making it so the order of constraints
* matter since the first pass through this will set the
* pn_min_memory based on that first constraint and if
* it isn't smaller than all the other requests they
* will fail. We have to keep track of the
* memory for accounting, these next 2 variables do this for us.
*/
uint64_t smallest_min_mem = INFINITE64;
uint64_t orig_req_mem = job_ptr->details->pn_min_memory;
if (test_only)
select_mode = SELECT_MODE_TEST_ONLY;
else
select_mode = SELECT_MODE_RUN_NOW;
if ((job_ptr->details->min_nodes == 0) &&
(job_ptr->details->max_nodes == 0)) {
/* Zero compute node job (burst buffer use only) */
avail_bitmap = bit_alloc(node_record_count);
pick_code = select_g_job_test(job_ptr,
avail_bitmap,
0, 0, 0,
select_mode,
preemptee_candidates,
preemptee_job_list,
resv_exc_ptr,
NULL);
if (pick_code == SLURM_SUCCESS) {
*select_bitmap = avail_bitmap;
return SLURM_SUCCESS;
} else {
FREE_NULL_BITMAP(avail_bitmap);
if (pick_code == ESLURM_LICENSES_UNAVAILABLE)
return ESLURM_LICENSES_UNAVAILABLE;
else
return ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE;
}
} else if (node_set_size == 0) {
info("%s: empty node set for selection", __func__);
return ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE;
}
shared = _resolve_shared_status(job_ptr, part_ptr->max_share);
/*
* If job preemption is enabled, then do NOT limit the set of available
* nodes by their current 'sharable' or 'idle' setting
*/
preempt_flag = slurm_preemption_enabled();
if (job_ptr->details->req_node_bitmap) { /* specific nodes required */
/*
* We have already confirmed that all of these nodes have a
* usable configuration and are in the proper partition.
* Check that these nodes can be used by this job.
*/
if (min_nodes != 0) {
total_nodes = bit_set_count(
job_ptr->details->req_node_bitmap);
}
if (total_nodes > max_nodes) { /* exceeds node limit */
return ESLURM_REQUESTED_PART_CONFIG_UNAVAILABLE;
}
if ((job_ptr->details->core_spec != NO_VAL16) &&
((job_ptr->details->core_spec & CORE_SPEC_THREAD) == 0)) {
i = bit_ffs(job_ptr->details->req_node_bitmap);
if (i >= 0) {
j = node_record_table_ptr[i]->tot_cores;
}
if ((i >= 0) && (job_ptr->details->core_spec >= j)) {
if (part_ptr->name) {
info("%s: %pJ never runnable in partition %s",
__func__, job_ptr,
part_ptr->name);
} else {
info("%s: %pJ never runnable",
__func__, job_ptr);
}
return ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE;
}
}
/*
* Check the availability of these nodes.
* Should we check memory availability on these nodes?
*/
if (!bit_super_set(job_ptr->details->req_node_bitmap,
avail_node_bitmap)) {
return ESLURM_NODE_NOT_AVAIL;
}
/*
* Still must go through select_g_job_test() to determine the
* validity of request and/or perform set-up before job launch
*/
total_nodes = 0; /* reinitialize */
}
/* identify the min and max feature values for possible exclusive OR */
max_feature = -1;
min_feature = MAX_FEATURES;
for (i = 0; i < node_set_size; i++) {
j = bit_ffs(node_set_ptr[i].feature_bits);
if ((j >= 0) && (j < min_feature))
min_feature = j;
j = bit_fls(node_set_ptr[i].feature_bits);
if ((j >= 0) && (j > max_feature))
max_feature = j;
}
debug3("%s: %pJ idle_nodes %u share_nodes %u",
__func__, job_ptr, bit_set_count(idle_node_bitmap),
bit_set_count(share_node_bitmap));
if (running_cons_tres())
_sync_node_weight(node_set_ptr, node_set_size);
/*
* Accumulate resources for this job based upon its required
* features (possibly with node counts).
*/
for (j = min_feature; j <= max_feature; j++) {
if (job_ptr->details->req_node_bitmap) {
bool missing_required_nodes = false;
bool feature_found = false;
for (i = 0; i < node_set_size; i++) {
if (!bit_test(node_set_ptr[i].feature_bits, j))
continue;
feature_found = true;
node_set_map =
bit_copy(node_set_ptr[i].my_bitmap);
if ((node_set_ptr[i].flags & NODE_SET_REBOOT)) {
/* Node reboot required */
bit_and(node_set_map,
idle_node_bitmap);
/*
* Powered up cloud nodes can't be
* rebooted to get new features. Must be
* powered down first.
*/
bit_and_not(node_set_map,
cloud_node_bitmap);
}
if (avail_bitmap) {
bit_or(avail_bitmap, node_set_map);
FREE_NULL_BITMAP(node_set_map);
} else {
avail_bitmap = node_set_map;
}
}
if (!feature_found)
continue;
if (!bit_super_set(job_ptr->details->req_node_bitmap,
avail_bitmap))
missing_required_nodes = true;
if (missing_required_nodes)
continue;
FREE_NULL_BITMAP(avail_bitmap);
avail_bitmap = bit_copy(job_ptr->details->
req_node_bitmap);
bit_and_not(avail_bitmap, rs_node_bitmap);
}
for (i = 0; i < node_set_size; i++) {
int count1 = 0, count2 = 0;
if (!has_xand &&
!bit_test(node_set_ptr[i].feature_bits, j)) {
if ((i+1) < node_set_size || !avail_bitmap)
continue;
else
goto try_sched;
}
if (total_bitmap) {
bit_or(total_bitmap,
node_set_ptr[i].my_bitmap);
} else {
total_bitmap = bit_copy(
node_set_ptr[i].my_bitmap);
}
if ((node_set_ptr[i].flags & NODE_SET_REBOOT)) {
/* Node reboot required */
count1 = bit_set_count(node_set_ptr[i].
my_bitmap);
bit_and(node_set_ptr[i].my_bitmap,
idle_node_bitmap);
/*
* Powered up cloud nodes can't be rebooted to
* get new features. Must be powered down first.
*/
bit_and_not(node_set_ptr[i].my_bitmap,
cloud_node_bitmap);
count2 = bit_set_count(node_set_ptr[i].
my_bitmap);
if (count1 != count2)
nodes_busy = true;
}
bit_and(node_set_ptr[i].my_bitmap, avail_node_bitmap);
if (!nodes_busy) {
count1 = bit_set_count(node_set_ptr[i].
my_bitmap);
}
if (!preempt_flag) {
if (shared) {
bit_and(node_set_ptr[i].my_bitmap,
share_node_bitmap);
bit_and_not(node_set_ptr[i].my_bitmap,
cg_node_bitmap);
} else {
bit_and(node_set_ptr[i].my_bitmap,
idle_node_bitmap);
/* IDLE nodes are not COMPLETING */
}
} else {
bit_and_not(node_set_ptr[i].my_bitmap,
cg_node_bitmap);
}
/*
* We must skip the node *only* in the case it is
* rebooted with ASAP flag.
*/
bit_and_not(node_set_ptr[i].my_bitmap,
asap_node_bitmap);
if (!nodes_busy) {
count2 = bit_set_count(node_set_ptr[i].
my_bitmap);
if (count1 != count2)
nodes_busy = true;
}
if (avail_bitmap) {
bit_or(avail_bitmap,
node_set_ptr[i].my_bitmap);
} else {
avail_bitmap = bit_copy(node_set_ptr[i].
my_bitmap);
}
tried_sched = false; /* need to test these nodes */
if (running_cons_tres() && ((i + 1) < node_set_size)) {
/*
* Execute select_g_job_test() _once_ using
* sched_weight in node_record_t as set
* by _sync_node_weight()
*/
continue;
}
try_sched:
/* NOTE: select_g_job_test() is destructive of
* avail_bitmap, so save a backup copy */
backup_bitmap = bit_copy(avail_bitmap);
FREE_NULL_LIST(*preemptee_job_list);
if (job_ptr->details->req_node_bitmap == NULL)
bit_and(avail_bitmap, avail_node_bitmap);
bit_and(avail_bitmap, share_node_bitmap);
avail_nodes = bit_set_count(avail_bitmap);
if (((avail_nodes < min_nodes) ||
((avail_nodes >= min_nodes) &&
(avail_nodes < req_nodes))) &&
((i+1) < node_set_size)) {
FREE_NULL_BITMAP(avail_bitmap);
avail_bitmap = backup_bitmap;
continue; /* Keep accumulating nodes */
}
/* Only preempt jobs when all possible nodes are being
* considered for use, otherwise we would preempt jobs
* to use the lowest weight nodes. */
if ((i+1) < node_set_size || !preemptee_candidates)
preemptee_cand = NULL;
else if (preempt_flag) {
job_record_t *tmp_job_ptr = NULL;
list_itr_t *job_iterator;
job_iterator = list_iterator_create(preemptee_candidates);
while ((tmp_job_ptr = list_next(job_iterator)))
_bit_or_cond(tmp_job_ptr, avail_bitmap);
list_iterator_destroy(job_iterator);
bit_and(avail_bitmap, avail_node_bitmap);
bit_and(avail_bitmap, total_bitmap);
preemptee_cand = preemptee_candidates;
} else
preemptee_cand = preemptee_candidates;
job_ptr->details->pn_min_memory = orig_req_mem;
pick_code = select_g_job_test(job_ptr,
avail_bitmap,
min_nodes,
max_nodes,
req_nodes,
select_mode,
preemptee_cand,
preemptee_job_list,
resv_exc_ptr,
NULL);
if (job_ptr->details->pn_min_memory) {
if (job_ptr->details->pn_min_memory <
smallest_min_mem)
smallest_min_mem =
job_ptr->details->pn_min_memory;
else
job_ptr->details->pn_min_memory =
smallest_min_mem;
}
#if _DEBUG
{
char *tmp_str1 = bitmap2node_name(avail_bitmap);
char *tmp_str2 = bitmap2node_name(backup_bitmap);
info("%s: %pJ err:%d nodes:%u:%u:%u mode:%u select %s from %s",
__func__, job_ptr, pick_code, min_nodes, req_nodes,
max_nodes, select_mode, tmp_str1, tmp_str2);
xfree(tmp_str1);
xfree(tmp_str2);
}
#endif
if (pick_code == SLURM_SUCCESS) {
FREE_NULL_BITMAP(backup_bitmap);
if (bit_set_count(avail_bitmap) > max_nodes) {
/* end of tests for this feature */
avail_nodes = 0;
break;
}
FREE_NULL_BITMAP(total_bitmap);
FREE_NULL_BITMAP(possible_bitmap);
*select_bitmap = avail_bitmap;
return SLURM_SUCCESS;
} else {
tried_sched = true; /* test failed */
FREE_NULL_BITMAP(avail_bitmap);
avail_bitmap = backup_bitmap;
}
} /* for (i = 0; i < node_set_size; i++) */
/* try to get req_nodes now for this feature */
if (avail_bitmap && (!tried_sched) &&
(avail_nodes >= min_nodes) &&
((job_ptr->details->req_node_bitmap == NULL) ||
bit_super_set(job_ptr->details->req_node_bitmap,
avail_bitmap))) {
FREE_NULL_LIST(*preemptee_job_list);
job_ptr->details->pn_min_memory = orig_req_mem;
pick_code = select_g_job_test(job_ptr, avail_bitmap,
min_nodes, max_nodes,
req_nodes,
select_mode,
preemptee_candidates,
preemptee_job_list,
resv_exc_ptr,
NULL);
if (job_ptr->details->pn_min_memory) {
if (job_ptr->details->pn_min_memory <
smallest_min_mem)
smallest_min_mem =
job_ptr->details->pn_min_memory;
else
job_ptr->details->pn_min_memory =
smallest_min_mem;
}
if ((pick_code == SLURM_SUCCESS) &&
(bit_set_count(avail_bitmap) <= max_nodes)) {
FREE_NULL_BITMAP(total_bitmap);
FREE_NULL_BITMAP(possible_bitmap);
*select_bitmap = avail_bitmap;
return SLURM_SUCCESS;
}
}
if (pick_code == ESLURM_LICENSES_UNAVAILABLE)
licenses_unavailable = true;
/* determine if job could possibly run (if all configured
* nodes available) */
if (total_bitmap)
total_nodes = bit_set_count(total_bitmap);
if (total_bitmap &&
(!runable_ever || !runable_avail) &&
(total_nodes >= min_nodes) &&
((job_ptr->details->req_node_bitmap == NULL) ||
(bit_super_set(job_ptr->details->req_node_bitmap,
total_bitmap)))) {
avail_nodes = bit_set_count(avail_bitmap);
if (!runable_avail && (avail_nodes >= min_nodes)) {
FREE_NULL_BITMAP(avail_bitmap);
avail_bitmap = bit_copy(total_bitmap);
bit_and(avail_bitmap, avail_node_bitmap);
job_ptr->details->pn_min_memory = orig_req_mem;
pick_code = select_g_job_test(job_ptr,
avail_bitmap,
min_nodes,
max_nodes,
req_nodes,
SELECT_MODE_TEST_ONLY,
preemptee_candidates, NULL,
resv_exc_ptr,
NULL);
if (job_ptr->details->pn_min_memory) {
if (job_ptr->details->pn_min_memory <
smallest_min_mem)
smallest_min_mem =
job_ptr->details->
pn_min_memory;
else
job_ptr->details->
pn_min_memory =
smallest_min_mem;
}
if (pick_code == SLURM_SUCCESS) {
runable_ever = true;
if (bit_set_count(avail_bitmap) <=
max_nodes)
runable_avail = true;
FREE_NULL_BITMAP(possible_bitmap);
possible_bitmap = avail_bitmap;
avail_bitmap = NULL;
}
}
if (!runable_ever) {
job_ptr->details->pn_min_memory = orig_req_mem;
pick_code = select_g_job_test(job_ptr,
total_bitmap,
min_nodes,
max_nodes,
req_nodes,
SELECT_MODE_TEST_ONLY,
preemptee_candidates, NULL,
resv_exc_ptr,
NULL);
if (job_ptr->details->pn_min_memory) {
if (job_ptr->details->pn_min_memory <
smallest_min_mem)
smallest_min_mem =
job_ptr->details->
pn_min_memory;
else
job_ptr->details->
pn_min_memory =
smallest_min_mem;
}
if (pick_code == SLURM_SUCCESS) {
FREE_NULL_BITMAP(possible_bitmap);
possible_bitmap = total_bitmap;
total_bitmap = NULL;
runable_ever = true;
}
}
}
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(total_bitmap);
if (error_code != SLURM_SUCCESS)
break;
}
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(total_bitmap);
/* The job is not able to start right now, return a
* value indicating when the job can start */
if (!runable_ever && resv_overlap &&
(pick_code != ESLURM_REQUESTED_TOPO_CONFIG_UNAVAILABLE)) {
error_code = ESLURM_RESERVATION_BUSY;
return error_code;
}
if (licenses_unavailable) {
error_code = ESLURM_LICENSES_UNAVAILABLE;
} else if (!runable_ever) {
char *tmp;
/*
* If a job requested extra_constraints, then assume
* that the job might be runnable at some point in the
* future. FIXME: This is a kludge and this assumption
* may be wrong.
*/
tmp = job_ptr->extra_constraints ?
"currently not runnable" : "never runnable";
if (part_ptr->name) {
info("%s: %pJ %s in partition %s",
__func__, job_ptr, tmp, part_ptr->name);
} else {
info("%s: job %pJ %s",
__func__, job_ptr, tmp);
}
if (pick_code == ESLURM_REQUESTED_TOPO_CONFIG_UNAVAILABLE)
error_code = pick_code;
else
error_code = ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE;
} else if (!runable_avail && !nodes_busy) {
error_code = ESLURM_NODE_NOT_AVAIL;
} else if (job_ptr->details->req_node_bitmap &&
bit_overlap_any(job_ptr->details->req_node_bitmap,
rs_node_bitmap)) {
error_code = ESLURM_NODES_BUSY;
} else if (!preempt_flag && job_ptr->details->req_node_bitmap) {
/* specific nodes required */
if (shared) {
if (!bit_super_set(job_ptr->details->req_node_bitmap,
share_node_bitmap)) {
error_code = ESLURM_NODES_BUSY;
}
if (bit_overlap_any(job_ptr->details->req_node_bitmap,
cg_node_bitmap)) {
error_code = ESLURM_NODES_BUSY;
}
} else if (!bit_super_set(job_ptr->details->req_node_bitmap,
idle_node_bitmap)) {
error_code = ESLURM_NODES_BUSY;
/* Note: IDLE nodes are not COMPLETING */
}
} else if (job_ptr->details->req_node_bitmap &&
bit_overlap_any(job_ptr->details->req_node_bitmap,
cg_node_bitmap)) {
error_code = ESLURM_NODES_BUSY;
}
if (error_code == SLURM_SUCCESS) {
error_code = ESLURM_NODES_BUSY;
}
if (possible_bitmap && runable_ever) {
*select_bitmap = possible_bitmap;
} else {
FREE_NULL_BITMAP(possible_bitmap);
}
return error_code;
}
static void _preempt_jobs(list_t *preemptee_job_list, bool kill_pending,
int *error_code, job_record_t *preemptor_ptr)
{
list_itr_t *iter;
job_record_t *job_ptr;
uint16_t mode;
int job_cnt = 0;
static time_t sched_update = 0;
if (sched_update != slurm_conf.last_update) {
preempt_send_user_signal = false;
if (xstrcasestr(slurm_conf.preempt_params,
"send_user_signal") ||
xstrcasestr(slurm_conf.slurmctld_params,
"preempt_send_user_signal"))
preempt_send_user_signal = true;
sched_update = slurm_conf.last_update;
}
iter = list_iterator_create(preemptee_job_list);
while ((job_ptr = list_next(iter))) {
mode = slurm_job_preempt_mode(job_ptr);
if (mode == PREEMPT_MODE_OFF) {
error("%s: Invalid preempt_mode %u for %pJ",
__func__, mode, job_ptr);
continue;
}
if ((mode == PREEMPT_MODE_SUSPEND) &&
(slurm_conf.preempt_mode & PREEMPT_MODE_GANG)) {
debug("preempted %pJ suspended by gang scheduler to reclaim resources for %pJ",
job_ptr, preemptor_ptr);
job_ptr->preempt_time = time(NULL);
continue;
}
job_cnt++;
if (!kill_pending)
continue;
if (slurm_job_preempt(job_ptr, preemptor_ptr, mode, true) !=
SLURM_SUCCESS)
continue;
}
list_iterator_destroy(iter);
if (job_cnt > 0)
*error_code = ESLURM_NODES_BUSY;
}
/* Return true if this job record is
* 1) not a job array OR
* 2) the first task of a job array to begin execution */
static bool _first_array_task(job_record_t *job_ptr)
{
job_record_t *meta_job_ptr;
if (job_ptr->array_task_id == NO_VAL)
return true;
meta_job_ptr = find_job_record(job_ptr->array_job_id);
if (!meta_job_ptr || !meta_job_ptr->array_recs) {
error("%s: Could not find meta job record for %pJ",
__func__, job_ptr);
return true;
}
if ((meta_job_ptr->array_recs->tot_run_tasks == 1) && /* This task */
(meta_job_ptr->array_recs->tot_comp_tasks == 0))
return true;
return false;
}
/*
* This job has zero node count. It is only designed to create or destroy
* persistent burst buffer resources. Terminate it now.
*/
static void _end_null_job(job_record_t *job_ptr)
{
time_t now = time(NULL);
job_ptr->exit_code = 0;
gres_stepmgr_job_clear_alloc(job_ptr->gres_list_req);
gres_stepmgr_job_clear_alloc(job_ptr->gres_list_req_accum);
FREE_NULL_LIST(job_ptr->gres_list_alloc);
job_state_set(job_ptr, JOB_RUNNING);
job_ptr->bit_flags |= JOB_WAS_RUNNING;
FREE_NULL_BITMAP(job_ptr->node_bitmap);
xfree(job_ptr->nodes);
xfree(job_ptr->sched_nodes);
job_ptr->start_time = now;
job_ptr->state_reason = WAIT_NO_REASON;
xfree(job_ptr->state_desc);
job_ptr->time_last_active = now;
if (!job_ptr->step_list)
job_ptr->step_list = list_create(free_step_record);
(void) job_array_post_sched(job_ptr, true);
(void) bb_g_job_begin(job_ptr);
job_array_start(job_ptr);
rebuild_job_part_list(job_ptr);
if ((job_ptr->mail_type & MAIL_JOB_BEGIN) &&
((job_ptr->mail_type & MAIL_ARRAY_TASKS) ||
_first_array_task(job_ptr)))
mail_job_info(job_ptr, MAIL_JOB_BEGIN);
slurmctld_diag_stats.jobs_started++;
/* Call job_set_alloc_tres() before acct_policy_job_begin() */
job_set_alloc_tres(job_ptr, false);
acct_policy_job_begin(job_ptr, false);
/*
* If run with slurmdbd, this is handled out of band in the job if
* happening right away. If the job has already become eligible and
* registered in the db then the start message.
*/
jobacct_storage_g_job_start(acct_db_conn, job_ptr);
jobcomp_g_record_job_start(job_ptr);
prolog_slurmctld(job_ptr);
job_ptr->end_time = now;
job_state_set(job_ptr, JOB_COMPLETE);
job_completion_logger(job_ptr, false);
acct_policy_job_fini(job_ptr, false);
if (select_g_job_fini(job_ptr) != SLURM_SUCCESS)
error("select_g_job_fini(%pJ): %m", job_ptr);
epilog_slurmctld(job_ptr);
}
static void _handle_explicit_req(void *x, void *arg)
{
gres_state_t *gres_state_job = x;
list_t **ret_gres_list = arg;
/* Copy over the explicit gres, skip others */
if (!(gres_state_job->config_flags & GRES_CONF_EXPLICIT) &&
!gres_id_shared(gres_state_job->config_flags))
return;
if (!*ret_gres_list)
*ret_gres_list = list_create(gres_job_list_delete);
list_append(*ret_gres_list,
gres_create_state(
gres_state_job,
GRES_STATE_SRC_STATE_PTR,
GRES_STATE_TYPE_JOB,
gres_job_state_dup(gres_state_job->gres_data)));
}
static void _gres_select_explicit(
list_t *req_gres_list, list_t **ret_gres_list)
{
if (!req_gres_list)
return;
(void) list_for_each(req_gres_list,
(ListForF) _handle_explicit_req,
ret_gres_list);
}
static list_t *_handle_exclusive_gres(job_record_t *job_ptr,
bitstr_t *select_bitmap, bool test_only)
{
list_t *post_list = NULL;
node_record_t *node_ptr;
if (test_only || !gres_get_gres_cnt())
return NULL;
xassert(job_ptr);
xassert(select_bitmap);
if (!job_ptr->details ||
!(job_ptr->details->whole_node & WHOLE_NODE_REQUIRED))
return NULL;
if (job_ptr->gres_list_req)
_gres_select_explicit(job_ptr->gres_list_req, &post_list);
for (int i = 0; (node_ptr = next_node_bitmap(select_bitmap, &i)); i++) {
gres_stepmgr_job_select_whole_node(
&post_list,
node_ptr->gres_list,
job_ptr->job_id,
node_ptr->name);
}
return post_list;
}
typedef struct {
uint64_t gpu_cnt;
int node_inx;
} foreach_node_gpu_args_t;
static int _get_node_gpu_sum(void *x, void *arg)
{
foreach_node_gpu_args_t *args = arg;
gres_state_t *gres_job_state = x;
gres_job_state_t *gres_js;
if (gres_job_state->plugin_id != gres_get_gpu_plugin_id())
return SLURM_SUCCESS;
gres_js = gres_job_state->gres_data;
args->gpu_cnt += gres_js->gres_cnt_node_select[args->node_inx];
return SLURM_SUCCESS;
}
static uint64_t _get_max_node_gpu_cnt(bitstr_t *node_bitmap, list_t* gres_list)
{
foreach_node_gpu_args_t args;
uint64_t max_node_gpu_cnt = 0;
xassert(node_bitmap);
xassert(gres_list);
for (int i = 0; (i = bit_ffs_from_bit(node_bitmap, i)) >= 0; i++) {
args.gpu_cnt = 0;
args.node_inx = i;
/* Get the sum of all gpu types on the node */
list_for_each(gres_list, _get_node_gpu_sum, &args);
max_node_gpu_cnt = MAX(max_node_gpu_cnt, args.gpu_cnt);
}
return max_node_gpu_cnt;
}
static int _get_resv_mpi_ports(job_record_t *job_ptr,
uint16_t *orig_resv_port_cnt,
uint32_t node_cnt,
time_t now)
{
int error_code = SLURM_SUCCESS;
bool resv_ports_present = false;
if (!(job_ptr->bit_flags & STEPMGR_ENABLED))
return SLURM_SUCCESS;
if (slurm_conf.mpi_params && xstrstr(slurm_conf.mpi_params, "ports="))
resv_ports_present = true;
if (resv_ports_present &&
(job_ptr->resv_port_cnt == NO_VAL16)) {
if (!job_ptr->job_resrcs) {
error("Select plugin failed to set job resources");
/*
* Do not attempt to allocate the select_bitmap nodes
* since select plugin failed to set job resources
*/
error_code = ESLURM_NODES_BUSY;
job_ptr->start_time = 0;
job_ptr->time_last_active = 0;
job_ptr->end_time = 0;
job_ptr->state_reason = WAIT_RESOURCES;
last_job_update = now;
xfree(job_ptr->state_desc);
return error_code;
}
*orig_resv_port_cnt = job_ptr->resv_port_cnt;
job_ptr->resv_port_cnt = 0;
/*
* reserved port count set to maximum task count on
* any node plus one, or if the job is exclusive give all
* resvered ports.
*/
if ((job_ptr->job_resrcs->node_req == NODE_CR_RESERVED) ||
(job_ptr->details->whole_node & WHOLE_NODE_REQUIRED)) {
job_ptr->resv_port_cnt =
resv_port_get_resv_port_cnt() - 1;
} else if (!job_ptr->details->overcommit &&
(job_ptr->details->num_tasks ||
job_ptr->details->ntasks_per_node ||
job_ptr->details->ntasks_per_tres)) {
for (int i = 0; i < node_cnt; i++) {
uint16_t tasks =
job_ptr->job_resrcs->tasks_per_node[i];
job_ptr->resv_port_cnt =
MAX(job_ptr->resv_port_cnt, tasks * 2);
}
} else if (!job_ptr->details->overcommit) {
uint16_t max_node_cpus = 0;
for (int i = 0; i < node_cnt; i++) {
max_node_cpus =
MAX(max_node_cpus,
job_ptr->job_resrcs->cpus[i] * 2);
}
job_ptr->resv_port_cnt = max_node_cpus;
} else if (job_ptr->details->ntasks_per_node) {
job_ptr->resv_port_cnt =
job_ptr->details->ntasks_per_node;
} else if (job_ptr->details->ntasks_per_tres &&
job_ptr->gres_list_req ) {
uint64_t max_gpu_per_node =
_get_max_node_gpu_cnt(
job_ptr->node_bitmap,
job_ptr->gres_list_req);
if (max_gpu_per_node > slurm_conf.max_tasks_per_node)
max_gpu_per_node =
slurm_conf.max_tasks_per_node;
job_ptr->resv_port_cnt =
(uint16_t) max_gpu_per_node *
job_ptr->details->ntasks_per_tres;
} else if (job_ptr->details->num_tasks) {
job_ptr->resv_port_cnt = ROUNDUP(
job_ptr->details->num_tasks, node_cnt);
} else {
job_ptr->resv_port_cnt = ROUNDUP(
job_ptr->job_resrcs->ncpus, node_cnt);
}
job_ptr->resv_port_cnt++;
}
if ((job_ptr->resv_port_cnt != NO_VAL16) &&
(job_ptr->resv_port_cnt != 0)) {
error_code = resv_port_job_alloc(job_ptr);
if (error_code) {
job_ptr->start_time = 0;
job_ptr->time_last_active = 0;
job_ptr->end_time = 0;
job_ptr->state_reason = WAIT_MPI_PORTS_BUSY;
last_job_update = now;
xfree(job_ptr->state_desc);
}
}
return error_code;
}
/*
* select_nodes - select and allocate nodes to a specific job
* IN job_node_select - pointer with at least a pointer to the job record
* IN test_only - if set do not allocate nodes, just confirm they
* could be allocated now
* IN select_node_bitmap - bitmap of nodes to be used for the
* job's resource allocation (not returned if NULL), caller
* must free
* IN submission - if set ignore reservations
* IN scheduler_type - which scheduler is calling this
* (i.e. SLURMDB_JOB_FLAG_BACKFILL, SLURMDB_JOB_FLAG_SCHED, etc)
* RET 0 on success, ESLURM code from slurm_errno.h otherwise
* globals: list_part - global list of partition info
* default_part_loc - pointer to default partition
* config_list - global list of node configuration info
* Notes: The algorithm is
* 1) Build a table (node_set_ptr) of nodes with the requisite
* configuration. Each table entry includes their weight,
* node_list, features, etc.
* 2) Call _pick_best_nodes() to select those nodes best satisfying
* the request, (e.g. best-fit or other criterion)
* 3) Call allocate_nodes() to perform the actual allocation
*/
extern int select_nodes(job_node_select_t *job_node_select,
bool test_only, bool submission,
uint32_t scheduler_type)
{
int bb, error_code = SLURM_SUCCESS, i, node_set_size = 0;
bitstr_t *select_bitmap = NULL;
struct node_set *node_set_ptr = NULL;
part_record_t *part_ptr = NULL;
uint8_t orig_whole_node, orig_share_res;
uint16_t orig_resv_port_cnt = 0;
uint32_t min_nodes = 0, max_nodes = 0, req_nodes = 0;
time_t now = time(NULL);
bool configuring = false;
list_t *preemptee_job_list = NULL;
uint32_t selected_node_cnt = NO_VAL;
uint64_t tres_req_cnt[slurmctld_tres_cnt];
bool can_reboot;
uint32_t qos_flags = 0;
assoc_mgr_lock_t qos_read_lock =
{ .assoc = READ_LOCK, .qos = READ_LOCK };
assoc_mgr_lock_t job_read_locks =
{ .assoc = READ_LOCK, .qos = WRITE_LOCK, .tres = READ_LOCK };
list_t *gres_list_pre = NULL;
bool gres_list_pre_set = false;
job_record_t *tmp_job, *job_ptr = job_node_select->job_ptr;
xassert(job_ptr);
xassert(job_ptr->magic == JOB_MAGIC);
/*
* The call path from _get_req_features() (called later in this
* function) can eventually call _resolve_shared_status(). This latter
* function can alter the job_ptr->details->{whole_node,share_res}.
*
* Saving the original values here and restoring them at cleanup time
* at the bottom of this function if needed.
*/
orig_whole_node = job_ptr->details->whole_node;
orig_share_res = job_ptr->details->share_res;
if (!acct_policy_job_runnable_pre_select(job_ptr, false))
return ESLURM_ACCOUNTING_POLICY;
part_ptr = job_ptr->part_ptr;
/* identify partition */
if (part_ptr == NULL) {
part_ptr = find_part_record(job_ptr->partition);
xassert(part_ptr);
job_ptr->part_ptr = part_ptr;
error("partition pointer reset for %pJ, part %s",
job_ptr, job_ptr->partition);
}
/* Quick check to see if this QOS is allowed on this partition. */
assoc_mgr_lock(&qos_read_lock);
if (job_ptr->qos_ptr)
qos_flags = job_ptr->qos_ptr->flags;
if ((error_code = part_policy_valid_qos(job_ptr->part_ptr,
job_ptr->qos_ptr,
job_ptr->user_id, job_ptr)) !=
SLURM_SUCCESS) {
assoc_mgr_unlock(&qos_read_lock);
return ESLURM_REQUESTED_PART_CONFIG_UNAVAILABLE;
}
/* Quick check to see if this account is allowed on this partition. */
if ((error_code = part_policy_valid_acct(
job_ptr->part_ptr,
job_ptr->assoc_ptr ? job_ptr->assoc_ptr->acct : NULL,
job_ptr))
!= SLURM_SUCCESS) {
assoc_mgr_unlock(&qos_read_lock);
return ESLURM_REQUESTED_PART_CONFIG_UNAVAILABLE;
}
assoc_mgr_unlock(&qos_read_lock);
/* Quick check to see if this group is allowed on this partition. */
if (!validate_group(job_ptr->part_ptr, job_ptr->user_id)) {
xfree(job_ptr->state_desc);
xstrfmtcat(job_ptr->state_desc,
"uid %u not in group permitted to use this partition (%s). groups allowed: %s",
job_ptr->user_id, job_ptr->part_ptr->name,
part_ptr->allow_groups);
debug2("%s: %s", __func__, job_ptr->state_desc);
job_ptr->state_reason = WAIT_ACCOUNT;
last_job_update = now;
return ESLURM_REQUESTED_PART_CONFIG_UNAVAILABLE;
}
if (job_ptr->priority == 0) { /* user/admin hold */
if (job_ptr->state_reason != FAIL_BAD_CONSTRAINTS
&& (job_ptr->state_reason != FAIL_BURST_BUFFER_OP)
&& (job_ptr->state_reason != WAIT_HELD)
&& (job_ptr->state_reason != WAIT_HELD_USER)
&& (job_ptr->state_reason != WAIT_MAX_REQUEUE)) {
job_ptr->state_reason = WAIT_HELD;
}
return ESLURM_JOB_HELD;
}
bb = bb_g_job_test_stage_in(job_ptr, test_only);
if (bb != 1) {
if ((bb == -1) &&
(job_ptr->state_reason == FAIL_BURST_BUFFER_OP))
return ESLURM_BURST_BUFFER_WAIT; /* Fatal BB event */
xfree(job_ptr->state_desc);
last_job_update = now;
if (bb == 0)
job_ptr->state_reason = WAIT_BURST_BUFFER_STAGING;
else
job_ptr->state_reason = WAIT_BURST_BUFFER_RESOURCE;
return ESLURM_BURST_BUFFER_WAIT;
}
if ((job_ptr->details->min_nodes == 0) &&
(job_ptr->details->max_nodes == 0)) {
if (!job_ptr->burst_buffer)
return ESLURM_INVALID_NODE_COUNT;
if (!test_only)
_end_null_job(job_ptr);
return SLURM_SUCCESS;
}
/* build sets of usable nodes based upon their configuration */
can_reboot = node_features_g_user_update(job_ptr->user_id);
error_code = _build_node_list(job_ptr, &node_set_ptr, &node_set_size,
job_node_select->err_msg,
test_only, can_reboot);
if (error_code)
return error_code;
if (node_set_ptr == NULL) /* Should never be true */
return ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE;
for (i = 0; i < node_set_size; i++)
_set_sched_weight(node_set_ptr + i);
qsort(node_set_ptr, node_set_size, sizeof(struct node_set),
_sort_node_set);
_log_node_set(job_ptr, node_set_ptr, node_set_size);
/* ensure that selected nodes are in these node sets */
if (job_ptr->details->req_node_bitmap) {
error_code = _nodes_in_sets(job_ptr->details->req_node_bitmap,
node_set_ptr, node_set_size);
if (error_code) {
info("No nodes satisfy requirements for %pJ in partition %s",
job_ptr, job_ptr->part_ptr->name);
goto cleanup;
}
}
/* enforce both user's and partition's node limits if the qos
* isn't set to override them */
/* info("req: %u-%u, %u", job_ptr->details->min_nodes, */
/* job_ptr->details->max_nodes, part_ptr->max_nodes); */
error_code = get_node_cnts(job_ptr, qos_flags, part_ptr,
&min_nodes, &req_nodes, &max_nodes);
if ((error_code == ESLURM_ACCOUNTING_POLICY) ||
(error_code == ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE))
goto cleanup;
else if ((error_code != ESLURM_REQUESTED_PART_CONFIG_UNAVAILABLE) &&
(error_code != ESLURM_RESERVATION_MAINT)) {
/* Select resources for the job here */
job_array_pre_sched(job_ptr);
if (job_ptr->job_resrcs)
debug2("%s: calling _get_req_features() for %pJ with not NULL job resources",
__func__, job_ptr);
error_code = _get_req_features(node_set_ptr, node_set_size,
&select_bitmap, job_ptr,
part_ptr, min_nodes, max_nodes,
req_nodes, test_only,
&preemptee_job_list, can_reboot,
submission);
}
/* Set this guess here to give the user tools an idea
* of how many nodes Slurm is planning on giving the job.
* This needs to be done on success or not. It means the job
* could run on nodes.
*/
if (select_bitmap) {
list_t *gres_list_whole_node = _handle_exclusive_gres(
job_ptr, select_bitmap, test_only);
selected_node_cnt = bit_set_count(select_bitmap);
job_ptr->node_cnt_wag = selected_node_cnt;
if (gres_list_whole_node) {
gres_list_pre_set = true;
gres_list_pre = job_ptr->gres_list_req;
job_ptr->gres_list_req = gres_list_whole_node;
}
} else
selected_node_cnt = req_nodes;
if (!test_only && select_bitmap && (max_powered_nodes != NO_VAL)) {
bitstr_t *tmp = bit_copy(select_bitmap);
hostlist_t *select = NULL, *need = NULL;
char *select_str = NULL, *need_str = NULL;
int32_t count, powerup_count, before_count = 0;
/* selected and powered down */
bit_and(tmp, power_down_node_bitmap);
powerup_count = bit_set_count(tmp);
if (slurm_conf.debug_flags & DEBUG_FLAG_POWER) {
select = bitmap2hostlist(select_bitmap);
select_str = slurm_hostlist_ranged_string_xmalloc(
select);
need = bitmap2hostlist(tmp);
need_str = slurm_hostlist_ranged_string_xmalloc(need);
before_count = bit_set_count(power_up_node_bitmap);
}
bit_or(tmp, power_up_node_bitmap);
count = bit_set_count(tmp);
log_flag(POWER, "Need to power up %d nodes (%s) from (%s). powered up count before: %d after: %d",
powerup_count, need_str, select_str, before_count,
count);
if ((powerup_count > 0) && (count > max_powered_nodes)) {
error_code = ESLURM_MAX_POWERED_NODES;
log_flag(POWER, "%s: Cannot power up more nodes for %pJ due to MaxPoweredUpNodes limit",
__func__, job_ptr);
}
FREE_NULL_BITMAP(tmp);
FREE_NULL_HOSTLIST(need);
FREE_NULL_HOSTLIST(select);
xfree(select_str);
xfree(need_str);
}
memcpy(tres_req_cnt, job_ptr->tres_req_cnt, sizeof(tres_req_cnt));
tres_req_cnt[TRES_ARRAY_CPU] =
(uint64_t)(job_ptr->total_cpus ?
job_ptr->total_cpus : job_ptr->details->min_cpus);
tres_req_cnt[TRES_ARRAY_MEM] = job_get_tres_mem(
job_ptr->job_resrcs,
job_ptr->details->pn_min_memory,
tres_req_cnt[TRES_ARRAY_CPU],
selected_node_cnt, job_ptr->part_ptr,
job_ptr->gres_list_req,
job_ptr->bit_flags & JOB_MEM_SET,
job_get_sockets_per_node(job_ptr),
job_ptr->details->num_tasks);
tres_req_cnt[TRES_ARRAY_NODE] = (uint64_t)selected_node_cnt;
assoc_mgr_lock(&job_read_locks);
gres_stepmgr_set_job_tres_cnt(
job_ptr->gres_list_req,
selected_node_cnt,
tres_req_cnt,
true);
tres_req_cnt[TRES_ARRAY_BILLING] =
assoc_mgr_tres_weighted(tres_req_cnt,
job_ptr->part_ptr->billing_weights,
slurm_conf.priority_flags, true);
if (!test_only && (selected_node_cnt != NO_VAL) &&
!acct_policy_job_runnable_post_select(job_ptr, tres_req_cnt, true)) {
assoc_mgr_unlock(&job_read_locks);
/* If there was an reason we couldn't schedule before hand we
* want to check if an accounting limit was also breached. If
* it was we want to override the other reason so if we are
* backfilling we don't reserve resources if we don't have to.
*/
free_job_resources(&job_ptr->job_resrcs);
if (error_code != SLURM_SUCCESS)
debug2("Replacing scheduling error code for %pJ from '%s' to 'Accounting policy'",
job_ptr, slurm_strerror(error_code));
error_code = ESLURM_ACCOUNTING_POLICY;
goto cleanup;
}
assoc_mgr_unlock(&job_read_locks);
/* set up the cpu_cnt here so we can decrement it as nodes
* free up. total_cpus is set within _get_req_features */
job_ptr->cpu_cnt = job_ptr->total_cpus;
if (!test_only && preemptee_job_list
&& (error_code == SLURM_SUCCESS)) {
job_details_t *detail_ptr = job_ptr->details;
time_t now = time(NULL);
bool kill_pending = true;
if ((detail_ptr->preempt_start_time != 0) &&
(detail_ptr->preempt_start_time >
(now - slurm_conf.kill_wait - slurm_conf.msg_timeout))) {
/* Job preemption may still be in progress,
* do not cancel or requeue any more jobs yet */
kill_pending = false;
}
_preempt_jobs(preemptee_job_list, kill_pending, &error_code,
job_ptr);
if ((error_code == ESLURM_NODES_BUSY) && kill_pending) {
detail_ptr->preempt_start_time = now;
job_ptr->preempt_in_progress = true;
if (job_ptr->array_recs)
job_ptr->array_recs->pend_run_tasks++;
}
}
if (error_code) {
/* Fatal errors for job here */
if (error_code == ESLURM_REQUESTED_PART_CONFIG_UNAVAILABLE) {
/* Too many nodes requested */
debug3("%s: %pJ not runnable with present config",
__func__, job_ptr);
job_ptr->state_reason = WAIT_PART_NODE_LIMIT;
xfree(job_ptr->state_desc);
last_job_update = now;
/* Non-fatal errors for job below */
} else if (error_code == ESLURM_NODE_NOT_AVAIL) {
/* Required nodes are down or drained */
char *node_str = NULL, *unavail_node = NULL;
bitstr_t *unavail_bitmap;
debug3("%s: %pJ required nodes not avail",
__func__, job_ptr);
job_ptr->state_reason = WAIT_NODE_NOT_AVAIL;
xfree(job_ptr->state_desc);
unavail_bitmap = bit_copy(avail_node_bitmap);
filter_by_node_owner(job_ptr, unavail_bitmap);
bit_not(unavail_bitmap);
bit_and_not(unavail_bitmap, future_node_bitmap);
bit_and(unavail_bitmap, part_ptr->node_bitmap);
bit_and_not(unavail_bitmap, up_node_bitmap);
if (job_ptr->details->req_node_bitmap) {
bit_and(unavail_bitmap,
job_ptr->details->req_node_bitmap);
}
if (bit_ffs(unavail_bitmap) != -1) {
unavail_node = bitmap2node_name(unavail_bitmap);
node_str = unavail_node;
}
FREE_NULL_BITMAP(unavail_bitmap);
if (node_str) {
xstrfmtcat(job_ptr->state_desc,
"ReqNodeNotAvail, "
"UnavailableNodes:%s",
node_str);
} else {
xstrfmtcat(job_ptr->state_desc,
"ReqNodeNotAvail, May be reserved "
"for other job");
}
xfree(unavail_node);
last_job_update = now;
} else if (error_code == ESLURM_RESERVATION_MAINT) {
error_code = ESLURM_RESERVATION_BUSY; /* All reserved */
job_ptr->state_reason = WAIT_NODE_NOT_AVAIL;
xfree(job_ptr->state_desc);
xstrfmtcat(job_ptr->state_desc,
"ReqNodeNotAvail, Reserved for maintenance");
} else if ((error_code == ESLURM_RESERVATION_NOT_USABLE) ||
(error_code == ESLURM_RESERVATION_BUSY)) {
job_ptr->state_reason = WAIT_RESERVATION;
xfree(job_ptr->state_desc);
} else if (error_code == ESLURM_LICENSES_UNAVAILABLE) {
job_ptr->state_reason = WAIT_LICENSES;
xfree(job_ptr->state_desc);
} else if ((job_ptr->state_reason == WAIT_HELD) &&
(job_ptr->priority == 0)) {
/* Held by select plugin due to some failure */
} else if ((error_code ==
ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE) &&
job_ptr->extra_constraints) {
/*
* If a job requested extra_constraints, then assume
* that the job might be runnable at some point in the
* future. FIXME: This is a kludge and this assumption
* may be wrong.
*/
job_ptr->state_reason = FAIL_CONSTRAINTS;
xfree(job_ptr->state_desc);
} else if (error_code == ESLURM_MAX_POWERED_NODES) {
job_ptr->state_reason = WAIT_MAX_POWERED_NODES;
xfree(job_ptr->state_desc);
} else {
job_ptr->state_reason = WAIT_RESOURCES;
xfree(job_ptr->state_desc);
}
goto cleanup;
}
if (test_only) { /* set if job not highest priority */
error_code = SLURM_SUCCESS;
goto cleanup;
}
/*
* This job may be getting requeued, clear vestigial state information
* before over-writing and leaking memory or referencing old GRES or
* step data.
*/
job_ptr->bit_flags &= ~JOB_KILL_HURRY;
job_state_unset_flag(job_ptr, JOB_POWER_UP_NODE);
FREE_NULL_BITMAP(job_ptr->node_bitmap);
xfree(job_ptr->nodes);
xfree(job_ptr->sched_nodes);
job_ptr->exit_code = 0;
gres_stepmgr_job_clear_alloc(job_ptr->gres_list_req);
gres_stepmgr_job_clear_alloc(job_ptr->gres_list_req_accum);
FREE_NULL_LIST(job_ptr->gres_list_alloc);
if (!job_ptr->step_list)
job_ptr->step_list = list_create(free_step_record);
job_ptr->node_bitmap = select_bitmap;
select_bitmap = NULL; /* nothing left to free */
if ((error_code = _get_resv_mpi_ports(job_ptr, &orig_resv_port_cnt,
selected_node_cnt, now)))
goto cleanup;
/*
* we need to have these times set to know when the endtime
* is for the job when we place it
*/
job_ptr->start_time = job_ptr->time_last_active = now;
if ((job_ptr->limit_set.time != ADMIN_SET_LIMIT) &&
((job_ptr->time_limit == NO_VAL) ||
((job_ptr->time_limit > part_ptr->max_time) &&
!(qos_flags & QOS_FLAG_PART_TIME_LIMIT)))) {
if (part_ptr->default_time != NO_VAL)
job_ptr->time_limit = part_ptr->default_time;
else
job_ptr->time_limit = part_ptr->max_time;
job_ptr->limit_set.time = 1;
}
job_end_time_reset(job_ptr);
/*
* job_array_post_sched() must happen before allocate_nodes() because
* we need the pending job array state to be copied. For example,
* allocate_nodes() calls license_job_get() which can modify the job's
* license_list if the job requested OR'd licenses.
*/
tmp_job = job_array_post_sched(job_ptr, true);
if (tmp_job && (tmp_job != job_ptr) && (orig_resv_port_cnt == NO_VAL16))
tmp_job->resv_port_cnt = orig_resv_port_cnt;
if (bb_g_job_begin(job_ptr) != SLURM_SUCCESS) {
/* Leave job queued, something is hosed */
error_code = ESLURM_INVALID_BURST_BUFFER_REQUEST;
error("bb_g_job_begin(%pJ): %s",
job_ptr, slurm_strerror(error_code));
job_ptr->start_time = 0;
job_ptr->time_last_active = 0;
job_ptr->end_time = 0;
job_ptr->priority = 0;
job_ptr->state_reason = WAIT_HELD;
last_job_update = now;
goto cleanup;
}
if (select_g_job_begin(job_ptr) != SLURM_SUCCESS) {
/* Leave job queued, something is hosed */
error("select_g_job_begin(%pJ): %m", job_ptr);
/* Cancel previously started job */
(void) bb_g_job_revoke_alloc(job_ptr);
error_code = ESLURM_NODES_BUSY;
job_ptr->start_time = 0;
job_ptr->time_last_active = 0;
job_ptr->end_time = 0;
job_ptr->state_reason = WAIT_RESOURCES;
last_job_update = now;
goto cleanup;
}
/* assign the nodes and stage_in the job */
job_ptr->state_reason = WAIT_NO_REASON;
xfree(job_ptr->state_desc);
if (job_ptr->job_resrcs && job_ptr->job_resrcs->nodes) {
job_ptr->nodes = xstrdup(job_ptr->job_resrcs->nodes);
} else {
error("Select plugin failed to set job resources, nodes");
/* Do not attempt to allocate the select_bitmap nodes since
* select plugin failed to set job resources */
/* Cancel previously started job */
(void) bb_g_job_revoke_alloc(job_ptr);
error_code = ESLURM_NODES_BUSY;
job_ptr->start_time = 0;
job_ptr->time_last_active = 0;
job_ptr->end_time = 0;
job_ptr->state_reason = WAIT_RESOURCES;
last_job_update = now;
goto cleanup;
}
job_ptr->db_flags &= ~SLURMDB_JOB_CLEAR_SCHED;
job_ptr->db_flags |= scheduler_type;
/* This could be set in the select plugin so we want to keep the flag */
configuring = IS_JOB_CONFIGURING(job_ptr);
job_state_set(job_ptr, JOB_RUNNING);
job_ptr->bit_flags |= JOB_WAS_RUNNING;
if (select_g_select_nodeinfo_set(job_ptr) != SLURM_SUCCESS) {
error("select_g_select_nodeinfo_set(%pJ): %m", job_ptr);
if (!job_ptr->job_resrcs) {
/* If we don't exit earlier the empty job_resrcs might
* be dereferenced later */
/* Cancel previously started job */
(void) bb_g_job_revoke_alloc(job_ptr);
error_code = ESLURM_NODES_BUSY;
job_ptr->start_time = 0;
job_ptr->time_last_active = 0;
job_ptr->end_time = 0;
job_ptr->state_reason = WAIT_RESOURCES;
job_state_set(job_ptr, JOB_PENDING);
last_job_update = now;
goto cleanup;
}
}
allocate_nodes(job_ptr);
job_array_start(job_ptr);
build_node_details(job_ptr, true);
rebuild_job_part_list(job_ptr);
if ((job_ptr->mail_type & MAIL_JOB_BEGIN) &&
((job_ptr->mail_type & MAIL_ARRAY_TASKS) ||
_first_array_task(job_ptr)))
mail_job_info(job_ptr, MAIL_JOB_BEGIN);
slurmctld_diag_stats.jobs_started++;
/* job_set_alloc_tres has to be done before acct_policy_job_begin */
job_set_alloc_tres(job_ptr, false);
acct_policy_job_begin(job_ptr, false);
resv_replace_update(job_ptr);
/*
* If ran with slurmdbd this is handled out of band in the
* job if happening right away. If the job has already
* become eligible and registered in the db then the start message.
*/
jobacct_storage_g_job_start(acct_db_conn, job_ptr);
jobcomp_g_record_job_start(job_ptr);
switch_g_job_start(job_ptr);
prolog_slurmctld(job_ptr);
reboot_job_nodes(job_ptr);
gs_job_start(job_ptr);
if (bit_overlap_any(job_ptr->node_bitmap, power_down_node_bitmap)) {
job_state_set_flag(job_ptr, JOB_POWER_UP_NODE);
if (resume_job_list) {
uint32_t *tmp = xmalloc(sizeof(uint32_t));
*tmp = job_ptr->job_id;
list_append(resume_job_list, tmp);
}
}
if (configuring || IS_JOB_POWER_UP_NODE(job_ptr) ||
!bit_super_set(job_ptr->node_bitmap, avail_node_bitmap)) {
/* This handles nodes explicitly requesting node reboot */
job_state_set_flag(job_ptr, JOB_CONFIGURING);
}
/*
* Request asynchronous launch of a prolog for a
* non-batch job as long as the node is not configuring for
* a reboot first. Job state could be changed above so we need to
* recheck its state to see if it's currently configuring.
* PROLOG_FLAG_CONTAIN also turns on PROLOG_FLAG_ALLOC.
*/
if (!IS_JOB_CONFIGURING(job_ptr)) {
if (slurm_conf.prolog_flags & PROLOG_FLAG_ALLOC)
launch_prolog(job_ptr);
}
cleanup:
if (job_ptr->array_recs && job_ptr->array_recs->task_id_bitmap &&
!IS_JOB_STARTED(job_ptr) &&
(bit_ffs(job_ptr->array_recs->task_id_bitmap) != -1)) {
job_ptr->array_task_id = NO_VAL;
}
FREE_NULL_LIST(preemptee_job_list);
FREE_NULL_BITMAP(select_bitmap);
if (node_set_ptr) {
for (i = 0; i < node_set_size; i++) {
xfree(node_set_ptr[i].features);
FREE_NULL_BITMAP(node_set_ptr[i].my_bitmap);
FREE_NULL_BITMAP(node_set_ptr[i].feature_bits);
}
xfree(node_set_ptr);
}
if (error_code != SLURM_SUCCESS) {
if (gres_list_pre_set &&
(job_ptr->gres_list_req != gres_list_pre)) {
FREE_NULL_LIST(job_ptr->gres_list_req);
job_ptr->gres_list_req = gres_list_pre;
}
if (orig_resv_port_cnt == NO_VAL16)
job_ptr->resv_port_cnt = orig_resv_port_cnt;
if (job_ptr->resv_ports) {
resv_port_job_free(job_ptr);
xfree(job_ptr->resv_ports);
}
FREE_NULL_BITMAP(job_ptr->node_bitmap);
} else
FREE_NULL_LIST(gres_list_pre);
/*
* Unless the job is allocated resources now, we need to restore the
* original whole_node/share_res values since _resolve_shared_status()
* might have altered them during evaluation, and we don't want to
* propagate the changes for potential subsequent evaluations for the
* same job in a different partition with different configuration.
*
* NOTE: If we ever add an early return between the call to
* _get_req_features() and the last return below we should ensure to
* amend the restore logic consequently (probably copy this snippet
* before such early return).
*
* NOTE: We could have moved this snippet right after the call to
* _get_req_features(), but we need it here since after the call the
* error_code might change.
*
* NOTE: select_nodes() is the first common caller ancestor of the
* different call tree ramifications ending in _resolve_shared_status(),
* thus considered the appropriate spot for the save/restore logic.
*/
if (test_only || (error_code != SLURM_SUCCESS)) {
job_ptr->details->whole_node = orig_whole_node;
job_ptr->details->share_res = orig_share_res;
}
return error_code;
}
/*
* get_node_cnts - determine the number of nodes for the requested job.
* IN job_ptr - pointer to the job record.
* IN qos_flags - Flags of the job_ptr's qos. This is so we don't have to send
* in a pointer or lock the qos read lock before calling.
* IN part_ptr - pointer to the job's partition.
* OUT min_nodes - The minimum number of nodes for the job.
* OUT req_nodes - The number of node the select plugin should target.
* OUT max_nodes - The max number of nodes for the job.
* RET SLURM_SUCCESS on success, ESLURM code from slurm_errno.h otherwise.
*/
extern int get_node_cnts(job_record_t *job_ptr, uint32_t qos_flags,
part_record_t *part_ptr, uint32_t *min_nodes,
uint32_t *req_nodes, uint32_t *max_nodes)
{
int error_code = SLURM_SUCCESS, i;
uint32_t acct_max_nodes;
uint32_t wait_reason = 0;
xassert(job_ptr);
xassert(part_ptr);
/* On BlueGene systems don't adjust the min/max node limits
* here. We are working on midplane values. */
if (qos_flags & QOS_FLAG_PART_MIN_NODE)
*min_nodes = job_ptr->details->min_nodes;
else
*min_nodes = MAX(job_ptr->details->min_nodes,
part_ptr->min_nodes);
if (!job_ptr->details->max_nodes)
*max_nodes = part_ptr->max_nodes;
else if (qos_flags & QOS_FLAG_PART_MAX_NODE)
*max_nodes = job_ptr->details->max_nodes;
else
*max_nodes = MIN(job_ptr->details->max_nodes,
part_ptr->max_nodes);
if (job_ptr->details->req_node_bitmap && job_ptr->details->max_nodes) {
i = bit_set_count(job_ptr->details->req_node_bitmap);
if (i > job_ptr->details->max_nodes) {
info("%pJ required node list has more nodes than the job can use (%d > %u)",
job_ptr, i, job_ptr->details->max_nodes);
error_code = ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE;
goto end_it;
}
}
/* Don't call functions in MIN/MAX it will result in the
* function being called multiple times. */
acct_max_nodes = acct_policy_get_max_nodes(job_ptr, &wait_reason);
*max_nodes = MIN(*max_nodes, acct_max_nodes);
*max_nodes = MIN(*max_nodes, 500000); /* prevent overflows */
if (!job_ptr->limit_set.tres[TRES_ARRAY_NODE] &&
job_ptr->details->max_nodes &&
!(job_ptr->bit_flags & USE_MIN_NODES))
*req_nodes = *max_nodes;
else
*req_nodes = *min_nodes;
if (acct_max_nodes < *min_nodes) {
error_code = ESLURM_ACCOUNTING_POLICY;
xfree(job_ptr->state_desc);
job_ptr->state_reason = wait_reason;
goto end_it;
} else if (*max_nodes < *min_nodes) {
error_code = ESLURM_REQUESTED_PART_CONFIG_UNAVAILABLE;
goto end_it;
}
end_it:
return error_code;
}
/*
* Launch prolog via RPC to slurmd. This is useful when we need to run
* prolog at allocation stage. Then we ask slurmd to launch the prolog
* asynchronously and wait on REQUEST_COMPLETE_PROLOG message from slurmd.
*/
extern void launch_prolog(job_record_t *job_ptr)
{
prolog_launch_msg_t *prolog_msg_ptr;
uint16_t protocol_version = job_ptr->start_protocol_ver;
uint16_t msg_flags = 0;
agent_arg_t *agent_arg_ptr;
job_resources_t *job_resrcs_ptr;
slurm_cred_arg_t cred_arg;
node_record_t *node_ptr;
xassert(job_ptr);
if (job_ptr->bit_flags & EXTERNAL_JOB)
return;
for (int i = 0; (node_ptr = next_node_bitmap(job_ptr->node_bitmap, &i));
i++) {
if (protocol_version > node_ptr->protocol_version)
protocol_version = node_ptr->protocol_version;
if (PACK_FANOUT_ADDRS(node_ptr))
msg_flags |= SLURM_PACK_ADDRS;
}
prolog_msg_ptr = xmalloc(sizeof(prolog_launch_msg_t));
/* Locks: Write job */
if ((slurm_conf.prolog_flags & PROLOG_FLAG_ALLOC) &&
!(slurm_conf.prolog_flags & PROLOG_FLAG_NOHOLD)) {
job_ptr->state_reason = WAIT_PROLOG;
FREE_NULL_BITMAP(job_ptr->node_bitmap_pr);
job_ptr->node_bitmap_pr = bit_copy(job_ptr->node_bitmap);
}
prolog_msg_ptr->alloc_tls_cert = xstrdup(job_ptr->alloc_tls_cert);
prolog_msg_ptr->job_gres_prep =
gres_g_prep_build_env(job_ptr->gres_list_alloc,
job_ptr->nodes);
prolog_msg_ptr->job_id = job_ptr->job_id;
prolog_msg_ptr->het_job_id = job_ptr->het_job_id;
prolog_msg_ptr->uid = job_ptr->user_id;
prolog_msg_ptr->gid = job_ptr->group_id;
if (!job_ptr->user_name)
job_ptr->user_name = user_from_job(job_ptr);
prolog_msg_ptr->nodes = xstrdup(job_ptr->nodes);
prolog_msg_ptr->work_dir = xstrdup(job_ptr->details->work_dir);
prolog_msg_ptr->x11 = job_ptr->details->x11;
if (prolog_msg_ptr->x11) {
char *x11_alloc_host = NULL;
prolog_msg_ptr->x11_magic_cookie =
xstrdup(job_ptr->details->x11_magic_cookie);
/*
* If resp_host is localhost, send slurmctld's hostname instead.
* This gives the compute node a better chance of getting the
* connection set up - otherwise it'd try to connect back to
* itself by mistake.
*/
if (!xstrncmp(job_ptr->resp_host, "127.", 4) ||
!xstrcmp(job_ptr->resp_host, "::1")) {
char hostname[HOST_NAME_MAX];
if (!gethostname(hostname, sizeof(hostname)))
x11_alloc_host = xstrdup(hostname);
}
if (!x11_alloc_host)
x11_alloc_host = xstrdup(job_ptr->resp_host);
prolog_msg_ptr->x11_alloc_host = x11_alloc_host;
prolog_msg_ptr->x11_alloc_port = job_ptr->other_port;
prolog_msg_ptr->x11_target = xstrdup(job_ptr->details->x11_target);
prolog_msg_ptr->x11_target_port = job_ptr->details->x11_target_port;
}
prolog_msg_ptr->spank_job_env_size = job_ptr->spank_job_env_size;
prolog_msg_ptr->spank_job_env = xduparray(job_ptr->spank_job_env_size,
job_ptr->spank_job_env);
if (job_ptr->bit_flags & STEPMGR_ENABLED) {
node_record_t *bit_node;
/* Only keep pointers to nodes */
list_t *job_node_array = list_create(NULL);
for (int i = 0;
(bit_node = next_node_bitmap(job_ptr->node_bitmap, &i));
i++) {
list_append(job_node_array, bit_node);
}
/*
* Pack while we are in locks so that we don't need to make a
* copies of job_ptr and job_node_array since the agent queue
* doesn't pack until sending.
*/
prolog_msg_ptr->job_ptr_buf = init_buf(BUF_SIZE);
job_record_pack(job_ptr, slurmctld_tres_cnt,
prolog_msg_ptr->job_ptr_buf, protocol_version);
prolog_msg_ptr->job_node_array_buf = init_buf(BUF_SIZE);
slurm_pack_list(job_node_array, node_record_pack,
prolog_msg_ptr->job_node_array_buf,
protocol_version);
prolog_msg_ptr->part_ptr_buf = init_buf(BUF_SIZE);
part_record_pack(job_ptr->part_ptr,
prolog_msg_ptr->part_ptr_buf,
protocol_version);
FREE_NULL_LIST(job_node_array);
}
xassert(job_ptr->job_resrcs);
job_resrcs_ptr = job_ptr->job_resrcs;
setup_cred_arg(&cred_arg, job_ptr);
cred_arg.step_id.job_id = job_ptr->job_id;
cred_arg.step_id.step_id = SLURM_EXTERN_CONT;
cred_arg.step_id.step_het_comp = NO_VAL;
if (job_resrcs_ptr->memory_allocated) {
slurm_array64_to_value_reps(job_resrcs_ptr->memory_allocated,
job_resrcs_ptr->nhosts,
&cred_arg.job_mem_alloc,
&cred_arg.job_mem_alloc_rep_count,
&cred_arg.job_mem_alloc_size);
}
cred_arg.step_core_bitmap = job_resrcs_ptr->core_bitmap;
cred_arg.step_hostlist = job_ptr->job_resrcs->nodes;
switch_g_extern_stepinfo(&cred_arg.switch_step, job_ptr);
prolog_msg_ptr->cred = slurm_cred_create(&cred_arg, false,
protocol_version);
switch_g_free_stepinfo(cred_arg.switch_step);
xfree(cred_arg.job_mem_alloc);
xfree(cred_arg.job_mem_alloc_rep_count);
if (!prolog_msg_ptr->cred) {
error("%s: slurm_cred_create failure for %pJ, holding job",
__func__, job_ptr);
slurm_free_prolog_launch_msg(prolog_msg_ptr);
job_mgr_handle_cred_failure(job_ptr);
return;
}
agent_arg_ptr = xmalloc(sizeof(agent_arg_t));
agent_arg_ptr->retry = 0;
agent_arg_ptr->protocol_version = protocol_version;
agent_arg_ptr->hostlist = hostlist_create(job_ptr->nodes);
agent_arg_ptr->node_count = job_ptr->node_cnt;
agent_arg_ptr->msg_type = REQUEST_LAUNCH_PROLOG;
agent_arg_ptr->msg_args = (void *) prolog_msg_ptr;
agent_arg_ptr->msg_flags = msg_flags;
/* At least on a Cray we have to treat this as a real step, so
* this is where to do it.
*/
if (slurm_conf.prolog_flags & PROLOG_FLAG_CONTAIN) {
step_record_t *step_ptr = build_extern_step(job_ptr);
if (!step_ptr)
error("%s: build_extern_step failure for %pJ",
__func__, job_ptr);
}
job_ptr->prolog_launch_time = time(NULL);
/* Launch the RPC via agent */
set_agent_arg_r_uid(agent_arg_ptr, SLURM_AUTH_UID_ANY);
agent_queue_request(agent_arg_ptr);
}
/*
* valid_feature_counts - validate a job's features can be satisfied
* by the selected nodes (NOTE: does not process MOR or XAND operators)
* IN job_ptr - job to operate on
* IN use_active - if set, then only consider nodes with the identified features
* active, otherwise use available features
* IN/OUT node_bitmap - nodes available for use, clear if unusable
* OUT has_mor - set if MOR/XAND found in feature expression
* RET SLURM_SUCCESS or error
*/
extern int valid_feature_counts(job_record_t *job_ptr, bool use_active,
bitstr_t *node_bitmap, bool *has_mor)
{
job_details_t *detail_ptr = job_ptr->details;
list_itr_t *job_feat_iter;
job_feature_t *job_feat_ptr;
int last_op = FEATURE_OP_AND, last_paren_op = FEATURE_OP_AND;
int last_paren_cnt = 0;
bitstr_t *feature_bitmap, *paren_bitmap = NULL;
bitstr_t *tmp_bitmap, *work_bitmap;
bool have_count = false, user_update;
int rc = SLURM_SUCCESS;
list_t *feature_list = NULL;
char *features;
xassert(detail_ptr);
xassert(node_bitmap);
xassert(has_mor);
/*
* This is used in two different ways. 1 to pick nodes where
* feature_use is set and another to set the predicted start time where
* it isn't.
*/
if (detail_ptr->features_use) {
feature_list = detail_ptr->feature_list_use;
features = detail_ptr->features_use;
} else {
feature_list = detail_ptr->feature_list;
features = detail_ptr->features;
}
*has_mor = false;
if (!feature_list) /* no constraints */
return rc;
user_update = node_features_g_user_update(job_ptr->user_id);
find_feature_nodes(feature_list, user_update);
feature_bitmap = bit_copy(node_bitmap);
work_bitmap = feature_bitmap;
job_feat_iter = list_iterator_create(feature_list);
while ((job_feat_ptr = list_next(job_feat_iter))) {
if (last_paren_cnt < job_feat_ptr->paren) {
/* Start of expression in parenthesis */
/*
* If this pair of parentheses is inside of brackets,
* then this is XAND or MOR. Set last_paren_op to
* avoid incorrectly doing bit_and() or bit_or() at the
* end of parentheses. This only matters if the
* parentheses are the first thing inside of brackets,
* in which case last_op is AND or OR depending on what
* (if anything) came before the brackets. If the
* parentheses are not the first thing inside of
* brackets then last_op is XAND or MOR.
*/
if (job_feat_ptr->bracket &&
(last_op != FEATURE_OP_XAND) &&
(last_op != FEATURE_OP_MOR))
last_paren_op = FEATURE_OP_XAND;
else
last_paren_op = last_op;
last_op = FEATURE_OP_AND;
if (paren_bitmap) {
if (job_ptr->job_id) {
error("%s: %pJ has bad feature expression: %s",
__func__, job_ptr,
features);
} else {
error("%s: Reservation has bad feature expression: %s",
__func__, features);
}
FREE_NULL_BITMAP(paren_bitmap);
}
paren_bitmap = bit_copy(node_bitmap);
work_bitmap = paren_bitmap;
}
if (use_active)
tmp_bitmap = job_feat_ptr->node_bitmap_active;
else
tmp_bitmap = job_feat_ptr->node_bitmap_avail;
if (tmp_bitmap) {
/*
* Here we need to use the current feature for MOR/AND
* not the last_op. For instance fastio&[xeon|nehalem]
* should ignore xeon (in valid_feature_count), but if
* would be based on last_op it will see AND operation.
* This should only be used when dealing with middle
* options, not for the end as done in the last_paren
* check below.
*/
if ((job_feat_ptr->op_code == FEATURE_OP_MOR) ||
(job_feat_ptr->op_code == FEATURE_OP_XAND)) {
*has_mor = true;
} else if (last_op == FEATURE_OP_AND) {
bit_and(work_bitmap, tmp_bitmap);
} else if (last_op == FEATURE_OP_OR) {
bit_or(work_bitmap, tmp_bitmap);
}
} else { /* feature not found */
if (last_op == FEATURE_OP_AND)
bit_clear_all(work_bitmap);
}
if (job_feat_ptr->count)
have_count = true;
if (last_paren_cnt > job_feat_ptr->paren) {
/* End of expression in parenthesis */
if (last_paren_op == FEATURE_OP_AND) {
bit_and(feature_bitmap, work_bitmap);
} else if (last_paren_op == FEATURE_OP_OR) {
bit_or(feature_bitmap, work_bitmap);
} else { /* FEATURE_OP_MOR or FEATURE_OP_XAND */
*has_mor = true;
}
FREE_NULL_BITMAP(paren_bitmap);
work_bitmap = feature_bitmap;
}
last_op = job_feat_ptr->op_code;
last_paren_cnt = job_feat_ptr->paren;
if (slurm_conf.debug_flags & DEBUG_FLAG_NODE_FEATURES) {
char *tmp_f, *tmp_w, *tmp_t;
tmp_f = bitmap2node_name(feature_bitmap);
tmp_w = bitmap2node_name(work_bitmap);
tmp_t = bitmap2node_name(tmp_bitmap);
log_flag(NODE_FEATURES, "%s: feature:%s feature_bitmap:%s work_bitmap:%s tmp_bitmap:%s count:%u",
__func__, job_feat_ptr->name, tmp_f, tmp_w,
tmp_t, job_feat_ptr->count);
xfree(tmp_f);
xfree(tmp_w);
xfree(tmp_t);
}
}
list_iterator_destroy(job_feat_iter);
if (!have_count)
bit_and(node_bitmap, work_bitmap);
FREE_NULL_BITMAP(feature_bitmap);
FREE_NULL_BITMAP(paren_bitmap);
if (slurm_conf.debug_flags & DEBUG_FLAG_NODE_FEATURES) {
char *tmp = bitmap2node_name(node_bitmap);
log_flag(NODE_FEATURES, "%s: NODES:%s HAS_MOR:%c status:%s",
__func__, tmp, (*has_mor ? 'T' : 'F'),
slurm_strerror(rc));
xfree(tmp);
}
return rc;
}
/*
* job_req_node_filter - job request node filter.
* clear from a bitmap the nodes which can not be used for a job
* test memory size, required features, processor count, etc.
* NOTE: Does not support exclusive OR of features.
* It just matches first element of MOR and ignores count.
* IN job_ptr - pointer to node to be scheduled
* IN/OUT bitmap - set of nodes being considered for use
* RET SLURM_SUCCESS or EINVAL if can't filter (exclusive OR of features)
*/
extern int job_req_node_filter(job_record_t *job_ptr,
bitstr_t *avail_bitmap, bool test_only)
{
job_details_t *detail_ptr = job_ptr->details;
multi_core_data_t *mc_ptr;
node_record_t *node_ptr;
bool has_mor = false;
if (detail_ptr == NULL) {
error("%s: %pJ has no details",
__func__, job_ptr);
return EINVAL;
}
mc_ptr = detail_ptr->mc_ptr;
for (int i = 0; (node_ptr = next_node_bitmap(avail_bitmap, &i)); i++) {
if ((detail_ptr->pn_min_cpus > node_ptr->cpus) ||
((detail_ptr->pn_min_memory & (~MEM_PER_CPU)) >
node_ptr->real_memory) ||
((detail_ptr->pn_min_memory & (MEM_PER_CPU)) &&
((detail_ptr->pn_min_memory & (~MEM_PER_CPU)) *
detail_ptr->pn_min_cpus) >
node_ptr->real_memory) ||
(detail_ptr->pn_min_tmp_disk >
node_ptr->tmp_disk)) {
bit_clear(avail_bitmap, i);
continue;
}
if (mc_ptr &&
(((mc_ptr->sockets_per_node > node_ptr->tot_sockets) &&
(mc_ptr->sockets_per_node != NO_VAL16)) ||
((mc_ptr->cores_per_socket > node_ptr->cores) &&
(mc_ptr->cores_per_socket != NO_VAL16)) ||
((mc_ptr->threads_per_core > node_ptr->threads) &&
(mc_ptr->threads_per_core != NO_VAL16)))) {
bit_clear(avail_bitmap, i);
continue;
}
}
return valid_feature_counts(job_ptr, false, avail_bitmap, &has_mor);
}
/*
* Split the node set record in two
* IN node_set_ptr - array of node_set records
* IN config_ptr - configuration info for the nodes being added to a node set
* IN nset_inx_base - index of original/base node_set to split
* IN nset_inx - index of the new node_set record
* IN nset_feature_bits - feature bitmap for the new node_set record
* IN nset_node_bitmap - bitmap of nodes for the new node_set record
* IN nset_flags - flags of nodes for the new node_set record
*/
static void _split_node_set(struct node_set *nset, config_record_t *config_ptr,
int nset_inx_base, int nset_inx,
bitstr_t *nset_feature_bits,
bitstr_t *nset_node_bitmap, uint32_t nset_flags)
{
nset[nset_inx].cpus_per_node = config_ptr->cpus;
nset[nset_inx].features = xstrdup(config_ptr->feature);
nset[nset_inx].feature_bits = bit_copy(nset_feature_bits);
nset[nset_inx].flags = nset_flags;
nset[nset_inx].real_memory = config_ptr->real_memory;
nset[nset_inx].node_weight = nset[nset_inx_base].node_weight;
/*
* The bitmap of this new nodeset will contain only the nodes that
* are present both in the original bitmap AND in the new bitmap.
*/
nset[nset_inx].my_bitmap = bit_copy(nset[nset_inx_base].my_bitmap);
bit_and(nset[nset_inx].my_bitmap, nset_node_bitmap);
nset[nset_inx].node_cnt = bit_set_count(nset[nset_inx].my_bitmap);
/* Now we remove these nodes from the original bitmap */
bit_and_not(nset[nset_inx_base].my_bitmap, nset_node_bitmap);
nset[nset_inx_base].node_cnt -= nset[nset_inx].node_cnt;
}
/* Split from an existing node_set */
static void _split_node_set2(struct node_set *nset, int idx, int *last_inx,
int cnt, bitstr_t *nset_bitmap,
uint32_t nset_flags)
{
nset[*last_inx].cpus_per_node = nset[idx].cpus_per_node;
nset[*last_inx].features = xstrdup(nset[idx].features);
nset[*last_inx].feature_bits = bit_copy(nset[idx].feature_bits);
nset[*last_inx].flags = nset_flags;
nset[*last_inx].real_memory = nset[idx].real_memory;
nset[*last_inx].node_weight = nset[idx].node_weight;
nset[*last_inx].my_bitmap = bit_copy(nset[idx].my_bitmap);
bit_and(nset[*last_inx].my_bitmap, nset_bitmap);
nset[*last_inx].node_cnt = cnt;
/* Remove the bits and count from the original set */
bit_and_not(nset[idx].my_bitmap, nset_bitmap);
nset[idx].node_cnt -= cnt;
(*last_inx)++;
}
static void _apply_extra_constraints(job_record_t *job_ptr,
bitstr_t *usable_node_mask)
{
node_record_t *node_ptr = NULL;
xassert(job_ptr->extra);
xassert(job_ptr->extra_constraints);
for (int i = 0; (node_ptr = next_node_bitmap(usable_node_mask, &i));
i++) {
if (!node_ptr->extra_data) {
bit_clear(usable_node_mask, i);
continue;
}
if (!extra_constraints_test(job_ptr->extra_constraints,
node_ptr->extra_data)) {
bit_clear(usable_node_mask, i);
continue;
}
}
}
/*
* _build_node_list - identify which nodes could be allocated to a job
* based upon node features, memory, processors, etc. Note that a
* bitmap is set to indicate which of the job's features that the
* nodes satisfy.
* IN job_ptr - pointer to node to be scheduled
* OUT node_set_pptr - list of node sets which could be used for the job
* OUT node_set_size - number of node_set entries
* OUT err_msg - error message for job, caller must xfree
* IN test_only - true if only testing if job can be started at some point
* IN can_reboot - if true node can use any available feature,
* else job can use only active features
* RET error code
*/
static int _build_node_list(job_record_t *job_ptr,
struct node_set **node_set_pptr,
int *node_set_size, char **err_msg, bool test_only,
bool can_reboot)
{
int adj_cpus, i, node_set_inx, node_set_len, node_set_inx_base;
int rc, qos_cnt;
struct node_set *node_set_ptr, *prev_node_set_ptr;
config_record_t *config_ptr;
part_record_t *part_ptr = job_ptr->part_ptr;
list_itr_t *config_iterator;
int total_cores;
job_details_t *detail_ptr = job_ptr->details;
bitstr_t *usable_node_mask = NULL;
multi_core_data_t *mc_ptr = detail_ptr->mc_ptr;
bitstr_t *tmp_feature;
bitstr_t *grp_node_bitmap;
bool has_mor = false;
bool resv_overlap = false;
bitstr_t *node_maps[NM_TYPES] = { NULL, NULL, NULL, NULL, NULL, NULL };
bitstr_t *reboot_bitmap = NULL;
if (job_ptr->resv_name) {
/*
* Limit node selection to those in selected reservation.
* Assume node reboot required since we have not selected the
* compute nodes yet.
*/
time_t start_res = time(NULL);
rc = job_test_resv(job_ptr, &start_res, false,
&usable_node_mask, NULL, &resv_overlap,
true);
if (rc != SLURM_SUCCESS) {
job_ptr->state_reason = WAIT_RESERVATION;
xfree(job_ptr->state_desc);
if (rc == ESLURM_INVALID_TIME_VALUE)
return ESLURM_RESERVATION_NOT_USABLE;
if (rc == ESLURM_NODES_BUSY)
return ESLURM_NODES_BUSY;
if (err_msg) {
xfree(*err_msg);
*err_msg = xstrdup("Problem using reservation");
}
return ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE;
}
if ((detail_ptr->req_node_bitmap) &&
(!bit_super_set(detail_ptr->req_node_bitmap,
usable_node_mask))) {
job_ptr->state_reason = WAIT_RESERVATION;
xfree(job_ptr->state_desc);
FREE_NULL_BITMAP(usable_node_mask);
if (err_msg) {
xfree(*err_msg);
*err_msg = xstrdup("Required nodes outside of "
"the reservation");
}
return ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE;
}
if (resv_overlap && bit_ffs(usable_node_mask) < 0) {
job_ptr->state_reason = WAIT_NODE_NOT_AVAIL;
xfree(job_ptr->state_desc);
xstrfmtcat(job_ptr->state_desc,
"ReqNodeNotAvail, Reserved for maintenance");
FREE_NULL_BITMAP(usable_node_mask);
return ESLURM_RESERVATION_BUSY; /* All reserved */
}
}
if (detail_ptr->exc_node_bitmap) {
if (usable_node_mask) {
bit_and_not(usable_node_mask, detail_ptr->exc_node_bitmap);
} else {
usable_node_mask =
bit_copy(detail_ptr->exc_node_bitmap);
bit_not(usable_node_mask);
}
} else if (usable_node_mask == NULL) {
usable_node_mask = node_conf_get_active_bitmap();
}
if (!(job_ptr->bit_flags & EXTERNAL_JOB)) {
bit_and_not(usable_node_mask, external_node_bitmap);
}
if (!test_only && job_ptr->extra_constraints) {
_apply_extra_constraints(job_ptr, usable_node_mask);
if (!bit_set_count(usable_node_mask)) {
rc = ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE;
debug("%s: No nodes satisfy %pJ extra constraints in partition %s",
__func__, job_ptr, job_ptr->part_ptr->name);
xfree(job_ptr->state_desc);
job_ptr->state_reason = FAIL_CONSTRAINTS;
debug2("%s: setting %pJ to \"%s\" (%s)",
__func__, job_ptr,
job_state_reason_string(job_ptr->state_reason),
slurm_strerror(rc));
FREE_NULL_BITMAP(usable_node_mask);
return rc;
}
}
if ((rc = valid_feature_counts(job_ptr, false, usable_node_mask,
&has_mor))) {
info("%pJ feature requirements can not be satisfied: %s",
job_ptr, slurm_strerror(rc));
FREE_NULL_BITMAP(usable_node_mask);
if (err_msg) {
xfree(*err_msg);
*err_msg = xstrdup("Node feature requirements can not "
"be satisfied");
}
return rc;
}
if (can_reboot)
reboot_bitmap = bit_alloc(node_record_count);
node_set_inx = 0;
node_set_len = list_count(config_list) * 32 + 1;
node_set_ptr = xcalloc(node_set_len, sizeof(struct node_set));
config_iterator = list_iterator_create(config_list);
while ((config_ptr = list_next(config_iterator))) {
bool cpus_ok = false, mem_ok = false, disk_ok = false;
bool job_mc_ok = false, config_filter = false;
total_cores = config_ptr->tot_sockets * config_ptr->cores;
adj_cpus = adjust_cpus_nppcu(_get_ntasks_per_core(detail_ptr),
detail_ptr->cpus_per_task,
total_cores, config_ptr->cpus);
if (detail_ptr->pn_min_cpus <= adj_cpus)
cpus_ok = true;
if ((detail_ptr->pn_min_memory & (~MEM_PER_CPU)) <=
config_ptr->real_memory)
mem_ok = true;
if (detail_ptr->pn_min_tmp_disk <= config_ptr->tmp_disk)
disk_ok = true;
if (!mc_ptr)
job_mc_ok = true;
if (mc_ptr &&
(((mc_ptr->sockets_per_node <= config_ptr->tot_sockets) ||
(mc_ptr->sockets_per_node == NO_VAL16)) &&
((mc_ptr->cores_per_socket <= config_ptr->cores) ||
(mc_ptr->cores_per_socket == NO_VAL16)) &&
((mc_ptr->threads_per_core <= config_ptr->threads) ||
(mc_ptr->threads_per_core == NO_VAL16))))
job_mc_ok = true;
config_filter = !(cpus_ok && mem_ok && disk_ok && job_mc_ok);
/*
* since nodes can register with more resources than defined
* in the configuration, we want to use those higher values
* for scheduling, but only as needed (slower)
*/
node_set_ptr[node_set_inx].my_bitmap =
bit_copy(config_ptr->node_bitmap);
bit_and(node_set_ptr[node_set_inx].my_bitmap,
part_ptr->node_bitmap);
if (usable_node_mask) {
bit_and(node_set_ptr[node_set_inx].my_bitmap,
usable_node_mask);
}
node_set_ptr[node_set_inx].node_cnt =
bit_set_count(node_set_ptr[node_set_inx].my_bitmap);
if (node_set_ptr[node_set_inx].node_cnt == 0) {
debug2("%s: JobId=%u matched 0 nodes (%s) due to job partition or features",
__func__, job_ptr->job_id, config_ptr->nodes);
FREE_NULL_BITMAP(node_set_ptr[node_set_inx].my_bitmap);
continue;
}
if (config_filter) {
_set_err_msg(cpus_ok, mem_ok, disk_ok, job_mc_ok,
err_msg);
debug2("%s: JobId=%u filtered all nodes (%s): %s",
__func__, job_ptr->job_id, config_ptr->nodes,
err_msg ? *err_msg : NULL);
FREE_NULL_BITMAP(node_set_ptr[node_set_inx].my_bitmap);
continue;
}
if (has_mor) {
tmp_feature = _valid_features(job_ptr, config_ptr,
can_reboot, reboot_bitmap);
if (tmp_feature == NULL) {
debug2("%s: JobId=%u matched 0 nodes (%s) due to MOR job features",
__func__, job_ptr->job_id,
config_ptr->nodes);
FREE_NULL_BITMAP(node_set_ptr[node_set_inx].
my_bitmap);
continue;
}
} else {
/* We've already filtered for AND/OR features */
tmp_feature = bit_alloc(MAX_FEATURES);
bit_set(tmp_feature, 0);
}
/* NOTE: FREE_NULL_BITMAP(tmp_feature) to avoid memory leak */
node_set_ptr[node_set_inx].cpus_per_node =
config_ptr->cpus;
node_set_ptr[node_set_inx].real_memory =
config_ptr->real_memory;
node_set_ptr[node_set_inx].node_weight = config_ptr->weight;
node_set_ptr[node_set_inx].features =
xstrdup(config_ptr->feature);
node_set_ptr[node_set_inx].feature_bits = tmp_feature;
debug2("found %u usable nodes from config containing %s",
node_set_ptr[node_set_inx].node_cnt, config_ptr->nodes);
prev_node_set_ptr = node_set_ptr + node_set_inx;
node_set_inx++;
if (node_set_inx >= node_set_len) {
error("%s: node_set buffer filled", __func__);
break;
}
/*
* If we have a FLEX reservation we will want a nodeset for
* those nodes outside the reservation.
*/
if (job_ptr->resv_ptr &&
(job_ptr->resv_ptr->flags & RESERVE_FLAG_FLEX) &&
job_ptr->resv_ptr->node_bitmap &&
!bit_super_set(prev_node_set_ptr->my_bitmap,
job_ptr->resv_ptr->node_bitmap)) {
node_maps[IN_FL] =
bit_copy(job_ptr->resv_ptr->node_bitmap);
node_maps[OUT_FL] =
bit_copy(prev_node_set_ptr->my_bitmap);
bit_and_not(node_maps[OUT_FL], node_maps[IN_FL]);
}
/* Identify the nodes that need reboot for use */
if (!test_only && can_reboot) {
if (has_mor) {
node_maps[REBOOT] = bit_copy(reboot_bitmap);
} else {
(void) _match_feature(
job_ptr->details->feature_list_use,
&node_maps[REBOOT]);
}
/* No nodes in set require reboot */
if (node_maps[REBOOT] &&
!bit_overlap_any(prev_node_set_ptr->my_bitmap,
node_maps[REBOOT]))
FREE_NULL_BITMAP(node_maps[REBOOT]);
}
/* No nodes to split from this node set */
if (!node_maps[OUT_FL] && !node_maps[REBOOT])
continue;
/* Just need to split these nodes that need reboot */
if (!node_maps[OUT_FL] && node_maps[REBOOT]) {
if (bit_super_set(prev_node_set_ptr->my_bitmap,
node_maps[REBOOT])) {
/* All nodes in set require reboot */
prev_node_set_ptr->flags = NODE_SET_REBOOT;
goto end_node_set;
}
node_set_inx_base = node_set_inx - 1;
_split_node_set(node_set_ptr, config_ptr,
node_set_inx_base, node_set_inx,
tmp_feature, node_maps[REBOOT],
NODE_SET_REBOOT);
node_set_inx++;
goto end_node_set;
}
/* Just need to split for these nodes that are outside FLEX */
if (node_maps[OUT_FL] && !node_maps[REBOOT]) {
if (bit_super_set(prev_node_set_ptr->my_bitmap,
node_maps[OUT_FL])) {
/* All nodes outside of flex reservation */
prev_node_set_ptr->flags =NODE_SET_OUTSIDE_FLEX;
goto end_node_set;
}
node_set_inx_base = node_set_inx - 1;
_split_node_set(node_set_ptr, config_ptr,
node_set_inx_base, node_set_inx,
tmp_feature, node_maps[OUT_FL],
NODE_SET_OUTSIDE_FLEX);
node_set_inx++;
goto end_node_set;
}
/* We may have to split in several subsets */
if (node_maps[OUT_FL] && node_maps[REBOOT]) {
node_maps[IN_FL_RE] = bit_copy(node_maps[IN_FL]);
bit_and(node_maps[IN_FL_RE], node_maps[REBOOT]);
node_maps[OUT_FL_RE] = bit_copy(node_maps[OUT_FL]);
bit_and(node_maps[OUT_FL_RE], node_maps[REBOOT]);
node_maps[OUT_FL_NO_RE] = bit_copy(node_maps[OUT_FL]);
bit_and_not(node_maps[OUT_FL_NO_RE],
node_maps[REBOOT]);
}
/*
* All nodes in this set should be avoided. No need to split.
* Just set the FLAGS and the Weight.
*/
if (bit_super_set(prev_node_set_ptr->my_bitmap,
node_maps[IN_FL_RE])) {
prev_node_set_ptr->flags = NODE_SET_REBOOT;
goto end_node_set;
}
if (bit_super_set(prev_node_set_ptr->my_bitmap,
node_maps[OUT_FL_NO_RE])) {
prev_node_set_ptr->flags = NODE_SET_OUTSIDE_FLEX;
goto end_node_set;
}
if (bit_super_set(prev_node_set_ptr->my_bitmap,
node_maps[OUT_FL_RE])) {
prev_node_set_ptr->flags = (NODE_SET_OUTSIDE_FLEX |
NODE_SET_REBOOT);
goto end_node_set;
}
/*
* At this point we split the node set record in four,
* in this order of priority:
*
* 1. Inside flex reservation and need to reboot
* 2. Outside flex reservation and NO need to reboot
* 3. Outside flex reservation and need to reboot
* 4. Available now, inside the flex reservation and NO need
* to reboot
*
* If there are no such reservations or need to reboot,
* additional nodesets will not be created.
*/
node_set_inx_base = node_set_inx - 1;
if (node_maps[IN_FL_RE]) {
_split_node_set(node_set_ptr, config_ptr,
node_set_inx_base, node_set_inx,
tmp_feature, node_maps[IN_FL_RE],
NODE_SET_REBOOT);
FREE_NULL_BITMAP(node_maps[IN_FL_RE]);
node_set_inx++;
if (node_set_inx >= node_set_len) {
error("%s: node_set buffer filled", __func__);
break;
}
}
if (node_maps[OUT_FL_NO_RE]) {
_split_node_set(node_set_ptr, config_ptr,
node_set_inx_base, node_set_inx,
tmp_feature, node_maps[OUT_FL_NO_RE],
(NODE_SET_OUTSIDE_FLEX));
FREE_NULL_BITMAP(node_maps[OUT_FL_NO_RE]);
node_set_inx++;
if (node_set_inx >= node_set_len) {
error("%s: node_set buffer filled", __func__);
break;
}
}
if (node_maps[OUT_FL_RE]) {
_split_node_set(node_set_ptr, config_ptr,
node_set_inx_base, node_set_inx,
tmp_feature, node_maps[OUT_FL_RE],
(NODE_SET_OUTSIDE_FLEX |
NODE_SET_REBOOT));
FREE_NULL_BITMAP(node_maps[OUT_FL_RE]);
node_set_inx++;
if (node_set_inx >= node_set_len) {
error("%s: node_set buffer filled", __func__);
break;
}
}
end_node_set:
for (i = 0; i < NM_TYPES; i++)
FREE_NULL_BITMAP(node_maps[i]);
if (node_set_inx >= node_set_len) {
error("%s: node_set buffer filled", __func__);
break;
}
}
list_iterator_destroy(config_iterator);
/* eliminate any incomplete node_set record */
xfree(node_set_ptr[node_set_inx].features);
FREE_NULL_BITMAP(node_set_ptr[node_set_inx].my_bitmap);
FREE_NULL_BITMAP(node_set_ptr[node_set_inx].feature_bits);
FREE_NULL_BITMAP(usable_node_mask);
if (node_set_inx == 0) {
rc = ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE;
info("%s: No nodes satisfy %pJ requirements in partition %s",
__func__, job_ptr, job_ptr->part_ptr->name);
xfree(node_set_ptr);
xfree(job_ptr->state_desc);
job_ptr->state_reason = FAIL_BAD_CONSTRAINTS;
debug2("%s: setting %pJ to \"%s\" (%s)",
__func__, job_ptr,
job_state_reason_string(job_ptr->state_reason),
slurm_strerror(rc));
FREE_NULL_BITMAP(reboot_bitmap);
return rc;
}
/*
* Clear message about any nodes which fail to satisfy specific
* job requirements as there are some nodes which can be used
*/
if (err_msg)
xfree(*err_msg);
/*
* If any nodes are powered down or powering up, put them into a
* new node_sets record with a higher scheduling weight. This means
* we avoid scheduling jobs on powered down and powering up nodes where
* possible. If those are required we prefer powering up nodes over
* powered down nodes.
*/
for (i = (node_set_inx - 1); i >= 0; i--) {
int booting_cnt = bit_overlap(node_set_ptr[i].my_bitmap,
booting_node_bitmap);
if (booting_cnt == 0)
continue; /* no nodes powering up */
if (booting_cnt == node_set_ptr[i].node_cnt) {
node_set_ptr[i].flags = NODE_SET_POWERING_UP;
continue; /* all nodes powering up */
}
/* Some nodes powering up, split record */
_split_node_set2(node_set_ptr, i, &node_set_inx, booting_cnt,
booting_node_bitmap, NODE_SET_POWERING_UP);
if (node_set_inx >= node_set_len) {
error("%s: node_set buffer filled", __func__);
break;
}
}
for (i = (node_set_inx-1); i >= 0; i--) {
int power_cnt = bit_overlap(node_set_ptr[i].my_bitmap,
power_down_node_bitmap);
if (power_cnt == 0)
continue; /* no nodes powered down */
if (power_cnt == node_set_ptr[i].node_cnt) {
node_set_ptr[i].flags = NODE_SET_POWER_DN;
continue; /* all nodes powered down */
}
/* Some nodes powered down, others up, split record */
_split_node_set2(node_set_ptr, i, &node_set_inx, power_cnt,
power_down_node_bitmap, NODE_SET_POWER_DN);
if (node_set_inx >= node_set_len) {
error("%s: node_set buffer filled", __func__);
break;
}
}
grp_node_bitmap = _find_grp_node_bitmap(job_ptr);
if (grp_node_bitmap) {
#if _DEBUG
char node_bitstr[64];
bit_fmt(node_bitstr, sizeof(node_bitstr), grp_node_bitmap);
info("%s: _find_grp_node_bitmap() grp_node_bitmap:%s", __func__, node_bitstr);
#endif
for (i = (node_set_inx-1); i >= 0; i--) {
qos_cnt = bit_overlap(node_set_ptr[i].my_bitmap,
grp_node_bitmap);
if (qos_cnt == 0) {
node_set_ptr[node_set_inx].node_weight += 1;
continue; /* no nodes overlap */
}
if (qos_cnt == node_set_ptr[i].node_cnt) {
continue; /* all nodes overlap */
}
/* Some nodes overlap, split record */
_split_node_set2(node_set_ptr, i, &node_set_inx,
qos_cnt, grp_node_bitmap,
node_set_ptr[i].flags);
node_set_ptr[i].node_weight++;
if (node_set_inx >= node_set_len) {
error("%s: node_set buffer filled", __func__);
break;
}
}
FREE_NULL_BITMAP(grp_node_bitmap);
}
FREE_NULL_BITMAP(reboot_bitmap);
*node_set_size = node_set_inx;
*node_set_pptr = node_set_ptr;
return SLURM_SUCCESS;
}
/*
* For a given node_set, set a scheduling weight based upon a combination of
* node_weight and flags (e.g. try to avoid reboot).
* 0x20000000000 - Requires boot
* 0x10000000000 - Outside of flex reservation
* 0x0########00 - Node weight
* 0x000000000## - Reserved for cons_tres, favor nodes with co-located CPU/GPU
*/
static void _set_sched_weight(struct node_set *node_set_ptr)
{
xassert(node_set_ptr);
node_set_ptr->sched_weight = node_set_ptr->node_weight << 8;
node_set_ptr->sched_weight |= 0xff;
if ((node_set_ptr->flags & NODE_SET_REBOOT) ||
(node_set_ptr->flags & NODE_SET_POWER_DN)) /* Boot required */
node_set_ptr->sched_weight |= 0x30000000000;
else if ((node_set_ptr->flags & NODE_SET_POWERING_UP))
node_set_ptr->sched_weight |= 0x20000000000;
else if (node_set_ptr->flags & NODE_SET_OUTSIDE_FLEX ||
node_set_ptr->flags & NODE_SET_POWERING_UP)
node_set_ptr->sched_weight |= 0x10000000000;
}
static int _sort_node_set(const void *x, const void *y)
{
struct node_set *node_set_ptr1 = (struct node_set *) x;
struct node_set *node_set_ptr2 = (struct node_set *) y;
xassert(node_set_ptr1);
xassert(node_set_ptr2);
if (node_set_ptr1->sched_weight < node_set_ptr2->sched_weight)
return -1;
if (node_set_ptr1->sched_weight > node_set_ptr2->sched_weight)
return 1;
return 0;
}
static void _log_node_set(job_record_t *job_ptr,
struct node_set *node_set_ptr,
int node_set_size)
{
char *node_list, feature_bits[64];
int i;
if (get_log_level() < LOG_LEVEL_DEBUG2)
return;
debug2("NodeSet for %pJ", job_ptr);
for (i = 0; i < node_set_size; i++) {
node_list = bitmap2node_name(node_set_ptr[i].my_bitmap);
if (node_set_ptr[i].feature_bits) {
bit_fmt(feature_bits, sizeof(feature_bits),
node_set_ptr[i].feature_bits);
} else
feature_bits[0] = '\0';
debug2("NodeSet[%d] Nodes:%s NodeWeight:%u Flags:%u FeatureBits:%s SchedWeight:%"PRIu64,
i, node_list, node_set_ptr[i].node_weight,
node_set_ptr[i].flags, feature_bits,
node_set_ptr[i].sched_weight);
xfree(node_list);
}
}
static void _set_err_msg(bool cpus_ok, bool mem_ok, bool disk_ok,
bool job_mc_ok, char **err_msg)
{
if (!err_msg)
return;
if (!cpus_ok) {
xfree(*err_msg);
*err_msg = xstrdup("CPU count per node can not be satisfied");
return;
}
if (!mem_ok) {
xfree(*err_msg);
*err_msg = xstrdup("Memory specification can not be satisfied");
return;
}
if (!disk_ok) {
xfree(*err_msg);
*err_msg = xstrdup("Temporary disk specification can not be "
"satisfied");
return;
}
if (!job_mc_ok) {
xfree(*err_msg);
*err_msg = xstrdup("Socket, core and/or thread specification "
"can not be satisfied");
return;
}
}
/*
* _nodes_in_sets - Determine if required nodes are included in node_set(s)
* IN req_bitmap - nodes specifically required by the job
* IN node_set_ptr - sets of valid nodes
* IN node_set_size - count of node_set entries
* RET 0 if in set, otherwise an error code
*/
static int _nodes_in_sets(bitstr_t *req_bitmap,
struct node_set * node_set_ptr,
int node_set_size)
{
bitstr_t *scratch_bitmap = NULL;
int error_code = SLURM_SUCCESS, i;
for (i=0; i<node_set_size; i++) {
if (scratch_bitmap)
bit_or(scratch_bitmap,
node_set_ptr[i].my_bitmap);
else {
scratch_bitmap =
bit_copy(node_set_ptr[i].my_bitmap);
}
}
if ((scratch_bitmap == NULL)
|| (bit_super_set(req_bitmap, scratch_bitmap) != 1))
error_code = ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE;
FREE_NULL_BITMAP(scratch_bitmap);
return error_code;
}
/*
* build_node_details - sets addresses for allocated nodes
* IN job_ptr - pointer to a job record
* IN new_alloc - set if new job allocation, cleared if state recovery
*/
extern void build_node_details(job_record_t *job_ptr, bool new_alloc)
{
hostlist_t *host_list = NULL;
node_record_t *node_ptr;
char *this_node_name;
int node_inx = 0;
if ((job_ptr->node_bitmap == NULL) || (job_ptr->nodes == NULL)) {
/* No nodes allocated, we're done... */
job_ptr->node_cnt = 0;
return;
}
/* Use hostlist here to ensure ordering of info matches that of srun */
if ((host_list = hostlist_create(job_ptr->nodes)) == NULL)
fatal("hostlist_create error for %s: %m", job_ptr->nodes);
job_ptr->total_nodes = job_ptr->node_cnt = hostlist_count(host_list);
xfree(job_ptr->batch_host);
while ((this_node_name = hostlist_shift(host_list))) {
if ((node_ptr = find_node_record(this_node_name))) {
node_inx++;
} else {
error("Invalid node %s in %pJ",
this_node_name, job_ptr);
}
if (!job_ptr->batch_host && !job_ptr->batch_features) {
/*
* Do not select until launch_job() as node features
* might be changed by node_features plugin between
* allocation time (now) and launch.
*/
job_ptr->batch_host = xstrdup(this_node_name);
}
free(this_node_name);
}
hostlist_destroy(host_list);
if (job_ptr->node_cnt != node_inx) {
error("Node count mismatch for %pJ (%u,%u)",
job_ptr, job_ptr->node_cnt, node_inx);
}
}
/*
* Set "batch_host" for this job based upon it's "batch_features" and
* "node_bitmap". Selection is performed on a best-effort basis (i.e. if no
* node satisfies the batch_features specification then pick first node).
* Execute this AFTER any node feature changes are made by the node_features
* plugin.
*
* If changes are made here, see if changes need to be made in
* test_job_nodes_ready().
*
* Return SLURM_SUCCESS or error code
*/
extern int pick_batch_host(job_record_t *job_ptr)
{
int i, i_first;
node_record_t *node_ptr;
char *tmp, *tok, sep, last_sep = '&';
node_feature_t *feature_ptr;
list_itr_t *feature_iter;
bitstr_t *feature_bitmap;
if (job_ptr->batch_host)
return SLURM_SUCCESS;
if (!job_ptr->node_bitmap) {
error("%s: %pJ lacks a node_bitmap", __func__, job_ptr);
return SLURM_ERROR;
}
i_first = bit_ffs(job_ptr->node_bitmap);
if (i_first < 0) {
error("%s: %pJ allocated no nodes", __func__, job_ptr);
return SLURM_ERROR;
}
if (!job_ptr->batch_features) {
/* Run batch script on first node of job allocation */
node_ptr = node_record_table_ptr[i_first];
job_ptr->batch_host = xstrdup(node_ptr->name);
return SLURM_SUCCESS;
}
feature_bitmap = bit_copy(job_ptr->node_bitmap);
tmp = xstrdup(job_ptr->batch_features);
tok = tmp;
for (i = 0; ; i++) {
if (tmp[i] == '&')
sep = '&';
else if (tmp[i] == '|')
sep = '|';
else if (tmp[i] == '\0')
sep = '\0';
else
continue;
tmp[i] = '\0';
feature_iter = list_iterator_create(active_feature_list);
while ((feature_ptr = list_next(feature_iter))) {
if (xstrcmp(feature_ptr->name, tok))
continue;
if (last_sep == '&') {
bit_and(feature_bitmap,
feature_ptr->node_bitmap);
} else {
bit_or(feature_bitmap,
feature_ptr->node_bitmap);
}
break;
}
list_iterator_destroy(feature_iter);
if (!feature_ptr) /* No match */
bit_clear_all(feature_bitmap);
if (sep == '\0')
break;
tok = tmp + i + 1;
last_sep = sep;
}
xfree(tmp);
bit_and(feature_bitmap, job_ptr->node_bitmap);
if ((i = bit_ffs(feature_bitmap)) >= 0)
node_ptr = node_record_table_ptr[i];
else
node_ptr = node_record_table_ptr[i_first];
job_ptr->batch_host = xstrdup(node_ptr->name);
FREE_NULL_BITMAP(feature_bitmap);
return SLURM_SUCCESS;
}
/*
* _valid_features - Determine if the requested features are satisfied by
* the available nodes. This is only used for MOR operators.
* IN job_ptr - job being scheduled
* IN config_ptr - node's configuration record
* IN can_reboot - if true node can use any available feature,
* else job can use only active features
* IN reboot_bitmap - bitmap of nodes requiring reboot for use (updated)
* RET NULL if request is not satisfied, otherwise a bitmap indicating
* which mutually exclusive features are satisfied. For example
* _valid_features("[fs1|fs2|fs3|fs4]", "fs3") returns a bitmap with
* the third bit set. For another example
* _valid_features("[fs1|fs2|fs3|fs4]", "fs1,fs3") returns a bitmap
* with the first and third bits set. The function returns a bitmap
* with the first bit set if requirements are satisfied without a
* mutually exclusive feature list.
*/
static bitstr_t *_valid_features(job_record_t *job_ptr,
config_record_t *config_ptr,
bool can_reboot, bitstr_t *reboot_bitmap)
{
job_details_t *details_ptr = job_ptr->details;
bitstr_t *result_node_bitmap = NULL, *paren_node_bitmap = NULL;
bitstr_t *working_node_bitmap, *active_node_bitmap = NULL;
bitstr_t *tmp_node_bitmap = NULL;
list_itr_t *feat_iter;
job_feature_t *job_feat_ptr;
int last_op = FEATURE_OP_AND, paren_op = FEATURE_OP_AND;
int last_paren = 0, position = 0;
if (details_ptr->feature_list_use == NULL) { /* no constraints */
result_node_bitmap = bit_alloc(MAX_FEATURES);
bit_set(result_node_bitmap, 0);
return result_node_bitmap;
}
feat_iter = list_iterator_create(details_ptr->feature_list_use);
while ((job_feat_ptr = list_next(feat_iter))) {
if (job_feat_ptr->paren > last_paren) {
/* Combine features within parenthesis */
paren_node_bitmap =
bit_copy(job_feat_ptr->node_bitmap_avail);
if (can_reboot)
active_node_bitmap = bit_copy(paren_node_bitmap);
last_paren = job_feat_ptr->paren;
paren_op = job_feat_ptr->op_code;
/*
* If this pair of parentheses is inside of brackets,
* then this is XAND or MOR. Set last_op so that the
* features in parentheses are considered as XAND or
* MOR and are evaluated in the if at the bottom of this
* loop. This only matters if the parentheses are the
* first thing inside of brackets because last_op is
* initialized to AND.
*/
if (job_feat_ptr->bracket &&
(last_op != FEATURE_OP_XAND) &&
(last_op != FEATURE_OP_MOR))
last_op = FEATURE_OP_XAND;
while ((job_feat_ptr = list_next(feat_iter))) {
if ((paren_op == FEATURE_OP_AND) &&
can_reboot) {
bit_and(paren_node_bitmap,
job_feat_ptr->node_bitmap_avail);
bit_and(active_node_bitmap,
job_feat_ptr->node_bitmap_active);
} else if (paren_op == FEATURE_OP_AND) {
bit_and(paren_node_bitmap,
job_feat_ptr->node_bitmap_active);
} else if ((paren_op == FEATURE_OP_OR) &&
can_reboot) {
bit_or(paren_node_bitmap,
job_feat_ptr->node_bitmap_avail);
bit_or(active_node_bitmap,
job_feat_ptr->node_bitmap_active);
} else if (paren_op == FEATURE_OP_OR) {
bit_or(paren_node_bitmap,
job_feat_ptr->node_bitmap_active);
} else {
error("%s: Bad feature expression for %pJ: %s",
__func__, job_ptr,
details_ptr->features_use);
break;
}
paren_op = job_feat_ptr->op_code;
if (job_feat_ptr->paren < last_paren) {
last_paren = job_feat_ptr->paren;
break;
}
}
working_node_bitmap = paren_node_bitmap;
} else {
working_node_bitmap = job_feat_ptr->node_bitmap_avail;
}
if (!job_feat_ptr) {
error("%s: Bad feature expression for %pJ: %s",
__func__, job_ptr, details_ptr->features_use);
}
if ((job_feat_ptr->op_code == FEATURE_OP_XAND) ||
(job_feat_ptr->op_code == FEATURE_OP_MOR) ||
((job_feat_ptr->op_code != FEATURE_OP_XAND) &&
(job_feat_ptr->op_code != FEATURE_OP_MOR) &&
((last_op == FEATURE_OP_XAND) ||
(last_op == FEATURE_OP_MOR)))) {
if (bit_overlap_any(config_ptr->node_bitmap,
working_node_bitmap)) {
if (!result_node_bitmap)
result_node_bitmap =
bit_alloc(MAX_FEATURES);
bit_set(result_node_bitmap, position);
if (can_reboot && reboot_bitmap &&
active_node_bitmap) {
tmp_node_bitmap = bit_copy(config_ptr->
node_bitmap);
bit_and_not(tmp_node_bitmap,
active_node_bitmap);
bit_or(reboot_bitmap, tmp_node_bitmap);
FREE_NULL_BITMAP(tmp_node_bitmap);
}
}
position++;
last_op = job_feat_ptr->op_code;
}
FREE_NULL_BITMAP(active_node_bitmap);
FREE_NULL_BITMAP(paren_node_bitmap);
}
list_iterator_destroy(feat_iter);
#if _DEBUG
{
char tmp[64];
if (result_node_bitmap)
bit_fmt(tmp, sizeof(tmp), result_node_bitmap);
else
snprintf(tmp, sizeof(tmp), "NONE");
info("CONFIG_FEATURE:%s FEATURE_MOR_BITS:%s", config_ptr->feature, tmp);
if (reboot_bitmap && (bit_ffs(reboot_bitmap) >= 0)) {
char *reboot_node_str = bitmap2node_name(reboot_bitmap);
info("REBOOT_NODES:%s", reboot_node_str);
xfree(reboot_node_str);
}
}
#endif
return result_node_bitmap;
}
/*
* re_kill_job - for a given job, deallocate its nodes for a second time,
* basically a cleanup for failed deallocate() calls
* IN job_ptr - pointer to terminating job (already in some COMPLETING state)
* globals: node_record_count - number of nodes in the system
* node_record_table_ptr - pointer to global node table
*/
extern void re_kill_job(job_record_t *job_ptr)
{
agent_arg_t *agent_args;
hostlist_t *kill_hostlist;
char *host_str = NULL;
static uint32_t last_job_id = 0;
node_record_t *node_ptr;
xassert(job_ptr);
xassert(job_ptr->details);
kill_hostlist = hostlist_create(NULL);
agent_args = xmalloc(sizeof(agent_arg_t));
agent_args->msg_type = REQUEST_TERMINATE_JOB;
agent_args->hostlist = hostlist_create(NULL);
agent_args->protocol_version = SLURM_PROTOCOL_VERSION;
agent_args->retry = 0;
if (job_ptr->node_bitmap_cg) {
for (int i = 0;
(node_ptr = next_node_bitmap(job_ptr->node_bitmap_cg, &i));
i++) {
if (IS_NODE_DOWN(node_ptr)) {
/* Consider job already completed */
bit_clear(job_ptr->node_bitmap_cg,
node_ptr->index);
job_update_tres_cnt(job_ptr, node_ptr->index);
if (node_ptr->comp_job_cnt)
(node_ptr->comp_job_cnt)--;
if ((job_ptr->node_cnt > 0) &&
((--job_ptr->node_cnt) == 0)) {
cleanup_completing(job_ptr, true);
last_node_update = time(NULL);
}
} else if (!IS_NODE_NO_RESPOND(node_ptr)) {
(void)hostlist_push_host(kill_hostlist,
node_ptr->name);
if (agent_args->protocol_version >
node_ptr->protocol_version)
agent_args->protocol_version =
node_ptr->protocol_version;
hostlist_push_host(agent_args->hostlist,
node_ptr->name);
agent_args->node_count++;
}
if (PACK_FANOUT_ADDRS(node_ptr))
agent_args->msg_flags |= SLURM_PACK_ADDRS;
}
}
if (agent_args->node_count == 0) {
FREE_NULL_HOSTLIST(agent_args->hostlist);
xfree(agent_args);
hostlist_destroy(kill_hostlist);
return;
}
hostlist_uniq(kill_hostlist);
host_str = hostlist_ranged_string_xmalloc(kill_hostlist);
if (job_ptr->job_id != last_job_id) {
info("Resending TERMINATE_JOB request %pJ Nodelist=%s",
job_ptr, host_str);
} else {
debug("Resending TERMINATE_JOB request %pJ Nodelist=%s",
job_ptr, host_str);
}
xfree(host_str);
last_job_id = job_ptr->job_id;
hostlist_destroy(kill_hostlist);
agent_args->msg_args =
create_kill_job_msg(job_ptr, agent_args->protocol_version);
set_agent_arg_r_uid(agent_args, SLURM_AUTH_UID_ANY);
agent_queue_request(agent_args);
}