| /*****************************************************************************\ | |
| * stepmgr.c - manage the job step information of slurm | |
| ***************************************************************************** | |
| * Copyright (C) 2002-2007 The Regents of the University of California. | |
| * Copyright (C) 2008-2010 Lawrence Livermore National Security. | |
| * Copyright (C) SchedMD LLC. | |
| * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). | |
| * Written by Morris Jette <jette1@llnl.gov>, et. al. | |
| * CODE-OCEC-09-009. All rights reserved. | |
| * | |
| * This file is part of Slurm, a resource management program. | |
| * For details, see <https://slurm.schedmd.com/>. | |
| * Please also read the included file: DISCLAIMER. | |
| * | |
| * Slurm is free software; you can redistribute it and/or modify it under | |
| * the terms of the GNU General Public License as published by the Free | |
| * Software Foundation; either version 2 of the License, or (at your option) | |
| * any later version. | |
| * | |
| * In addition, as a special exception, the copyright holders give permission | |
| * to link the code of portions of this program with the OpenSSL library under | |
| * certain conditions as described in each individual source file, and | |
| * distribute linked combinations including the two. You must obey the GNU | |
| * General Public License in all respects for all of the code used other than | |
| * OpenSSL. If you modify file(s) with this exception, you may extend this | |
| * exception to your version of the file(s), but you are not obligated to do | |
| * so. If you do not wish to do so, delete this exception statement from your | |
| * version. If you delete this exception statement from all source files in | |
| * the program, then also delete it here. | |
| * | |
| * Slurm is distributed in the hope that it will be useful, but WITHOUT ANY | |
| * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | |
| * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more | |
| * details. | |
| * | |
| * You should have received a copy of the GNU General Public License along | |
| * with Slurm; if not, write to the Free Software Foundation, Inc., | |
| * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | |
| \*****************************************************************************/ | |
| #include "config.h" | |
| #include <ctype.h> | |
| #include <errno.h> | |
| #include <signal.h> | |
| #include <stdio.h> | |
| #include <stdlib.h> | |
| #include <string.h> | |
| #include <sys/param.h> | |
| #include <sys/types.h> | |
| #include <time.h> | |
| #include <unistd.h> | |
| #include "stepmgr.h" | |
| #include "slurm/slurm_errno.h" | |
| #include "src/common/assoc_mgr.h" | |
| #include "src/common/bitstring.h" | |
| #include "src/common/dependency.h" | |
| #include "src/common/forward.h" | |
| #include "src/common/node_conf.h" | |
| #include "src/common/node_features.h" | |
| #include "src/common/port_mgr.h" | |
| #include "src/common/slurm_protocol_pack.h" | |
| #include "src/common/slurm_protocol_socket.h" | |
| #include "src/common/slurm_resource_info.h" | |
| #include "src/common/step_ctx.h" | |
| #include "src/common/stepd_proxy.h" | |
| #include "src/common/tres_bind.h" | |
| #include "src/common/tres_frequency.h" | |
| #include "src/common/xstring.h" | |
| #include "src/interfaces/accounting_storage.h" | |
| #include "src/interfaces/gres.h" | |
| #include "src/interfaces/jobacct_gather.h" | |
| #include "src/interfaces/mcs.h" | |
| #include "src/interfaces/mpi.h" | |
| #include "src/interfaces/select.h" | |
| #include "src/interfaces/switch.h" | |
| #include "src/interfaces/topology.h" | |
| #include "src/stepmgr/gres_stepmgr.h" | |
| #include "src/stepmgr/srun_comm.h" | |
| #include "src/stepmgr/step_dependency.h" | |
| typedef struct { | |
| uint16_t flags; | |
| bool found; | |
| char *kill_user; | |
| int rc_in; | |
| uint16_t signal; | |
| slurm_step_id_t step_id; | |
| } step_signal_t; | |
| typedef struct { | |
| bitstr_t *all_gres_core_bitmap; | |
| bitstr_t *any_gres_core_bitmap; | |
| int core_end_bit; | |
| int core_start_bit; | |
| int job_node_inx; | |
| list_t *node_gres_list; | |
| } foreach_gres_filter_t; | |
| /* Own new dependency edges and borrow an accepted placeholder during create. */ | |
| typedef struct { | |
| list_t *depend_list; /* owned until transferred */ | |
| bool dependency_reconsider; /* defer dependency reevaluation */ | |
| char *expression; /* borrowed from the request, not owned */ | |
| bool or_flag; | |
| step_record_t *pending_step; /* borrowed until create resolves */ | |
| } step_depend_ctx_t; | |
| /* Arguments to the pending-step and duplicate-ID lookup. */ | |
| typedef struct { | |
| step_record_t *pending_step; | |
| step_create_req_msg_t *step_specs; | |
| } step_create_match_t; | |
| static node_rank_order_t *_build_arbitrary_order(job_resources_t | |
| *job_resrcs_ptr, | |
| hostlist_t *dedup_nodes_hl, | |
| int *order_cnt); | |
| static step_record_t *_build_pending_step(job_record_t *job_ptr, | |
| step_create_req_msg_t *step_specs, | |
| uint16_t protocol_version); | |
| static int _step_partial_comp(step_record_t *step_ptr, | |
| step_complete_msg_t *req, bool finish, | |
| int *rem, uint32_t *max_rc); | |
| static int _count_cpus(job_record_t *job_ptr, bitstr_t *bitmap, | |
| uint32_t *usable_cpu_cnt); | |
| static void _dump_step_layout(step_record_t *step_ptr); | |
| static bool _is_mem_resv(void); | |
| static int _opt_cpu_cnt(uint32_t step_min_cpus, bitstr_t *node_bitmap, | |
| uint32_t *usable_cpu_cnt); | |
| static int _opt_node_cnt(uint32_t step_min_nodes, uint32_t step_max_nodes, | |
| int nodes_avail, int nodes_picked_cnt); | |
| static bitstr_t *_pick_step_nodes(job_record_t *job_ptr, | |
| step_create_req_msg_t *step_spec, | |
| list_t *step_gres_list, int cpus_per_task, | |
| uint32_t node_count, int *return_code); | |
| static bitstr_t *_pick_step_nodes_cpus(job_record_t *job_ptr, | |
| bitstr_t *nodes_bitmap, int node_cnt, | |
| int cpu_cnt, uint32_t *usable_cpu_cnt); | |
| static void _step_dealloc_lps(step_record_t *step_ptr); | |
| static int _step_create(slurm_msg_t *msg, job_record_t *job_ptr, | |
| step_create_req_msg_t *step_specs, | |
| step_record_t **new_step_record, | |
| step_record_t *wake_step, bool *dependency_reconsider, | |
| char **err_msg); | |
| static step_record_t *_build_interactive_step(job_record_t *job_ptr_in, | |
| step_create_req_msg_t *step_specs, | |
| uint16_t protocol_version); | |
| static int _build_ext_launcher_step(step_record_t **new_step_record, | |
| job_record_t *job_ptr, | |
| step_create_req_msg_t *step_specs, | |
| uint16_t protocol_version); | |
| static void _wake_pending_steps(job_record_t *job_ptr); | |
| static int _reap_failed_push_step(void *x, void *arg); | |
| static int _make_step_cred(step_record_t *step_ptr, slurm_cred_t **slurm_cred, | |
| uint16_t protocol_version); | |
| stepmgr_ops_t *stepmgr_ops = NULL; | |
| extern void stepmgr_init(stepmgr_ops_t *ops) | |
| { | |
| /* | |
| * Just keep pointers so that the pointers can be assigned after being | |
| * initialized init is called. | |
| */ | |
| stepmgr_ops = ops; | |
| slurm_daemon |= WITH_STEPMGR; | |
| } | |
| /* Determine how many more CPUs are required for a job step */ | |
| static int _opt_cpu_cnt(uint32_t step_min_cpus, bitstr_t *node_bitmap, | |
| uint32_t *usable_cpu_cnt) | |
| { | |
| int rem_cpus = step_min_cpus; | |
| if (!node_bitmap) | |
| return rem_cpus; | |
| xassert(usable_cpu_cnt); | |
| for (int i = 0; next_node_bitmap(node_bitmap, &i); i++) { | |
| if (usable_cpu_cnt[i] >= rem_cpus) | |
| return 0; | |
| rem_cpus -= usable_cpu_cnt[i]; | |
| } | |
| return rem_cpus; | |
| } | |
| /* Select the optimal node count for a job step based upon it's min and | |
| * max target, available resources, and nodes already picked */ | |
| static int _opt_node_cnt(uint32_t step_min_nodes, uint32_t step_max_nodes, | |
| int nodes_avail, int nodes_picked_cnt) | |
| { | |
| int target_node_cnt; | |
| if ((step_max_nodes > step_min_nodes) && (step_max_nodes != NO_VAL)) | |
| target_node_cnt = step_max_nodes; | |
| else | |
| target_node_cnt = step_min_nodes; | |
| if (target_node_cnt > nodes_picked_cnt) | |
| target_node_cnt -= nodes_picked_cnt; | |
| else | |
| target_node_cnt = 0; | |
| if (nodes_avail < target_node_cnt) | |
| target_node_cnt = nodes_avail; | |
| return target_node_cnt; | |
| } | |
| /* | |
| * _complete_unlaunched_step - Complete the accounting row of a step that | |
| * never launched: a pending placeholder, or a provisional step reaped | |
| * before its push. Only force the terminal state; the exit code stays | |
| * unset, which is how the drained push tells never-launched from a step | |
| * that exited. The accounting plugin skips the job-row derived_ec | |
| * update for an unset exit code. Only a placeholder that reserved a | |
| * real id was recorded at build, so the complete is skipped for a | |
| * SLURM_PENDING_STEP id. | |
| * IN/OUT step_ptr - unlaunched step being torn down | |
| */ | |
| static void _complete_unlaunched_step(step_record_t *step_ptr) | |
| { | |
| step_ptr->state = JOB_CANCELLED; | |
| if (step_ptr->step_id.step_id != SLURM_PENDING_STEP) | |
| jobacct_storage_g_step_complete(stepmgr_ops->acct_db_conn, | |
| step_ptr); | |
| } | |
| /* | |
| * Tear down a pending step whose dependencies can never be satisfied as | |
| * never launched, as _complete_unlaunched_step() does, but record FAILED | |
| * rather than CANCELLED. The placeholder has no layout or resources for the | |
| * running-step completion to release. | |
| * IN/OUT step_ptr - unlaunched step being torn down | |
| */ | |
| static void _fail_unlaunched_step(step_record_t *step_ptr) | |
| { | |
| step_ptr->state = JOB_FAILED; | |
| if (step_ptr->step_id.step_id != SLURM_PENDING_STEP) | |
| jobacct_storage_g_step_complete(stepmgr_ops->acct_db_conn, | |
| step_ptr); | |
| } | |
| /* | |
| * Find a re-sent pending step or a conflicting step ID without removing it. | |
| * IN x - step record to match | |
| * IN/OUT arg - request and context receiving the pending step | |
| * RET nonzero if the request matches this record | |
| */ | |
| static int _find_step_for_create(void *x, void *arg) | |
| { | |
| step_record_t *step_ptr = x; | |
| step_create_match_t *args = arg; | |
| step_create_req_msg_t *step_specs = args->step_specs; | |
| if (IS_STEP_PENDING(step_ptr) && !(step_ptr->flags & SSF_ASYNC) && | |
| (step_ptr->srun_pid == step_specs->srun_pid) && | |
| !xstrcmp(step_ptr->host, step_specs->host)) { | |
| args->pending_step = step_ptr; | |
| return 1; | |
| } | |
| /* | |
| * See if we have the same step id. If we do check to see if we | |
| * have the same step_het_comp or if the step's is NO_VAL, | |
| * meaning this step is not a het step. | |
| */ | |
| if ((step_specs->step_id.step_id == step_ptr->step_id.step_id) && | |
| ((step_specs->step_id.step_het_comp == | |
| step_ptr->step_id.step_het_comp) || | |
| (step_ptr->step_id.step_het_comp == NO_VAL))) | |
| return 1; | |
| return 0; | |
| } | |
| static int _set_step_id(step_record_t *step_ptr, | |
| step_create_req_msg_t *step_specs) | |
| { | |
| job_record_t *job_ptr = step_ptr->job_ptr; | |
| /* | |
| * Do not use STEP_ID_FROM_JOB_PTR here. | |
| * The step_specs layout is already set up in the exact | |
| * way required for HetStep launches. | |
| */ | |
| step_ptr->step_id = step_specs->step_id; | |
| step_ptr->step_id.sluid = job_ptr->step_id.sluid; | |
| if (step_specs->array_task_id != NO_VAL) | |
| step_ptr->step_id.job_id = job_ptr->job_id; | |
| if (step_specs->step_id.step_id != NO_VAL) { | |
| if (step_specs->step_id.step_het_comp == NO_VAL) { | |
| job_ptr->next_step_id = | |
| MAX(job_ptr->next_step_id, | |
| step_specs->step_id.step_id); | |
| job_ptr->next_step_id++; | |
| } | |
| } else if (job_ptr->het_job_id && | |
| (job_ptr->het_job_id != job_ptr->job_id)) { | |
| uint32_t *id_out = &step_ptr->step_id.step_id; | |
| int rc; | |
| rc = stepmgr_ops->get_het_step_id(job_ptr->het_job_id, id_out); | |
| if (rc != SLURM_SUCCESS) | |
| return rc; | |
| job_ptr->next_step_id = | |
| MAX(job_ptr->next_step_id, step_ptr->step_id.step_id); | |
| } else { | |
| step_ptr->step_id.step_id = job_ptr->next_step_id++; | |
| } | |
| return SLURM_SUCCESS; | |
| } | |
| static slurm_step_ctx_t *_step_ctx_create_stepmgr(step_create_req_msg_t | |
| *step_req, | |
| step_create_resp_msg_t | |
| *step_resp, | |
| step_record_t *step_rec) | |
| { | |
| slurm_step_ctx_t *ctx = xmalloc(sizeof(struct slurm_step_ctx_struct)); | |
| ctx->launch_state = NULL; | |
| ctx->magic = STEP_CTX_MAGIC; | |
| ctx->job_id = step_rec->step_id.job_id; | |
| ctx->step_resp = step_resp; | |
| ctx->step_req = step_req; | |
| ctx->launch_state = step_launch_state_create(ctx); | |
| return ctx; | |
| } | |
| /* | |
| * Trust the certificate the requesting srun sent for its step listener. | |
| * The job's certificate belongs to whatever created the allocation (NULL for | |
| * batch jobs), not to this srun, so it cannot verify this srun's listener. | |
| */ | |
| static void _set_step_tls_cert(step_record_t *step_ptr, | |
| step_create_req_msg_t *step_specs) | |
| { | |
| if (!step_specs->tls_cert) | |
| return; | |
| xfree(step_ptr->alloc_tls_cert); | |
| step_ptr->alloc_tls_cert = xstrdup(step_specs->tls_cert); | |
| } | |
| /* | |
| * Create a JOB_PENDING placeholder recording the host/port used to wake a | |
| * pending srun once another step ends. | |
| * | |
| * Terminology: a synchronous step (the srun default) blocks its srun until | |
| * the step launches; an asynchronous step (SSF_ASYNC, i.e. srun --async) is | |
| * fire-and-forget. Either kind can be queued here as a JOB_PENDING | |
| * placeholder while it waits for resources. | |
| * IN job_ptr - job to add the placeholder under | |
| * IN/OUT step_specs - step create request; for a >= 26.11 peer its step_id is | |
| * set to the assigned id and echoed back (and, in the stepmgr, the | |
| * request is retained on the placeholder for create-at-wake), else left | |
| * unchanged | |
| * IN protocol_version - create request's negotiated version; gates whether a | |
| * real id is assigned now (and the request retained) or the | |
| * SLURM_PENDING_STEP sentinel is kept. | |
| * When 26.05 is no longer supported remove this parameter. | |
| * RET placeholder on success, NULL otherwise | |
| */ | |
| static step_record_t *_build_pending_step(job_record_t *job_ptr, | |
| step_create_req_msg_t *step_specs, | |
| uint16_t protocol_version) | |
| { | |
| step_record_t *step_ptr; | |
| if ((!step_specs->host || !step_specs->port) && | |
| !(step_specs->flags & SSF_ASYNC)) | |
| return NULL; | |
| step_ptr = create_step_record(job_ptr, 0); | |
| if (step_ptr == NULL) | |
| return NULL; | |
| *stepmgr_ops->last_job_update = time(NULL); | |
| if (step_specs->flags & SSF_ASYNC) { | |
| if (_set_step_id(step_ptr, step_specs)) { | |
| delete_step_record(job_ptr, step_ptr); | |
| return NULL; | |
| } | |
| } | |
| step_ptr->cpu_count = step_specs->num_tasks; | |
| step_ptr->cwd = xstrdup(step_specs->cwd); | |
| step_ptr->flags = step_specs->flags; | |
| step_ptr->host = xstrdup(step_specs->host); | |
| _set_step_tls_cert(step_ptr, step_specs); | |
| step_ptr->srun_pid = step_specs->srun_pid; | |
| step_ptr->state = JOB_PENDING; | |
| step_ptr->std_err = xstrdup(step_specs->std_err); | |
| step_ptr->std_in = xstrdup(step_specs->std_in); | |
| step_ptr->std_out = xstrdup(step_specs->std_out); | |
| step_ptr->submit_line = xstrdup(step_specs->submit_line); | |
| if (step_specs->flags & SSF_ASYNC) { | |
| step_ptr->step_req = step_specs; | |
| step_specs->step_id = step_ptr->step_id; | |
| step_ptr->name = xstrdup(step_specs->name); | |
| jobacct_storage_g_step_start(stepmgr_ops->acct_db_conn, | |
| step_ptr); | |
| } else { | |
| /* | |
| * Assign a real id only when the peer can round-trip it on a | |
| * re-send; otherwise keep the sentinel and the id-at-launch | |
| * behavior (an old srun re-sends NO_VAL and would relaunch | |
| * with a different id). | |
| */ | |
| if (protocol_version >= SLURM_26_11_PROTOCOL_VERSION) { | |
| /* | |
| * Retain step_req and the srun's negotiated version so | |
| * the wake can create the step under the lock and push | |
| * it to this srun (create-at-wake). Only the stepmgr | |
| * pushes, so the controller would carry (and state | |
| * save) a request it can never use. free_step_record() | |
| * frees step_req if the placeholder is reaped first. | |
| */ | |
| if (!running_in_slurmctld()) { | |
| step_ptr->step_req = step_specs; | |
| step_ptr->start_protocol_ver = protocol_version; | |
| } | |
| /* A re-queue re-sends its already-assigned id. */ | |
| if (step_specs->step_id.step_id == NO_VAL) { | |
| if (_set_step_id(step_ptr, step_specs)) { | |
| /* The caller still owns step_specs */ | |
| step_ptr->step_req = NULL; | |
| delete_step_record(job_ptr, step_ptr); | |
| return NULL; | |
| } | |
| step_specs->step_id = step_ptr->step_id; | |
| } else { | |
| step_ptr->step_id = step_specs->step_id; | |
| step_ptr->step_id.sluid = | |
| job_ptr->step_id.sluid; | |
| if (step_specs->array_task_id != NO_VAL) | |
| step_ptr->step_id.job_id = | |
| job_ptr->job_id; | |
| } | |
| /* | |
| * A real id was assigned: record the placeholder as | |
| * PENDING, matching async, so a step that reserved an id | |
| * is never invisible to accounting. The re-start when the | |
| * step is created for real updates the row's name and | |
| * start time. | |
| */ | |
| step_ptr->name = xstrdup(step_specs->name); | |
| jobacct_storage_g_step_start(stepmgr_ops->acct_db_conn, | |
| step_ptr); | |
| } else { | |
| step_ptr->step_id = STEP_ID_FROM_JOB_RECORD(job_ptr); | |
| step_ptr->step_id.step_id = SLURM_PENDING_STEP; | |
| } | |
| step_ptr->port = step_specs->port; | |
| } | |
| if (job_ptr->node_bitmap) | |
| step_ptr->step_node_bitmap = bit_copy(job_ptr->node_bitmap); | |
| step_ptr->time_last_active = time(NULL); | |
| return step_ptr; | |
| } | |
| /* | |
| * Transfer newly validated dependencies to a pending step. | |
| * IN/OUT dep_ctx - the create's dependency context, emptied by the transfer | |
| * IN/OUT step_ptr - the record to take ownership | |
| */ | |
| static void _transfer_step_dependency(step_depend_ctx_t *dep_ctx, | |
| step_record_t *step_ptr) | |
| { | |
| if (!dep_ctx || !dep_ctx->depend_list) | |
| return; | |
| xassert(!step_ptr->step_depend_list); | |
| step_ptr->step_dependency = xstrdup(dep_ctx->expression); | |
| step_ptr->step_depend_list = dep_ctx->depend_list; | |
| step_ptr->step_depend_or = dep_ctx->or_flag; | |
| dep_ctx->depend_list = NULL; | |
| } | |
| /* | |
| * Queue a placeholder for a step that failed to schedule on a busy resource | |
| * and report it as queued. The placeholder is built on the initial create | |
| * only; wake and re-send attempts already have one. | |
| * IN job_ptr - job the step belongs to | |
| * IN/OUT step_specs - create request; gains an assigned step_id when queued | |
| * IN protocol_version - negotiated create-request version. | |
| * When 26.05 is no longer supported remove this parameter. | |
| * IN busy_errno - the busy failure being handled; ESLURM_STEP_DEP_NOT_MET | |
| * records the placeholder as waiting on a dependency, not resources | |
| * IN from_wake - true when recreating a step from its own pending placeholder, | |
| * which already holds the request | |
| * IN/OUT dep_ctx - accepted pending step or dependencies for a new placeholder | |
| * RET ESLURM_STEP_QUEUED if queued, else busy_errno unchanged | |
| */ | |
| static int _queue_pending_step(job_record_t *job_ptr, | |
| step_create_req_msg_t *step_specs, | |
| uint16_t protocol_version, int busy_errno, | |
| bool from_wake, step_depend_ctx_t *dep_ctx) | |
| { | |
| step_record_t *step_ptr = dep_ctx->pending_step; | |
| /* | |
| * Synchronous immediate requests need a placeholder while srun waits | |
| * out its window; srun cancels it when it stops waiting. | |
| */ | |
| if (!from_wake && !step_ptr && | |
| !((step_specs->flags & SSF_ASYNC) && step_specs->immediate)) { | |
| step_ptr = _build_pending_step(job_ptr, step_specs, | |
| protocol_version); | |
| if (step_ptr) | |
| _transfer_step_dependency(dep_ctx, step_ptr); | |
| } | |
| if (step_ptr) { | |
| if (busy_errno == ESLURM_STEP_DEP_NOT_MET) | |
| step_ptr->state_reason = WAIT_DEPENDENCY; | |
| else | |
| step_ptr->state_reason = WAIT_RESOURCES; | |
| } | |
| /* | |
| * Report queued only when a placeholder owns an id the peer can | |
| * re-send: the one just built, or the one this recreate came from. | |
| */ | |
| if ((step_ptr || from_wake) && | |
| ((protocol_version >= SLURM_26_11_PROTOCOL_VERSION) || | |
| (step_specs->flags & SSF_ASYNC)) && | |
| (step_specs->step_id.step_id != NO_VAL)) | |
| return ESLURM_STEP_QUEUED; | |
| return busy_errno; | |
| } | |
| /* | |
| * Publish a step lifecycle event without waking or deleting steps. | |
| * IN step_ptr - the step whose outcome this is | |
| * IN started - true for a committed start, false for a terminal outcome | |
| * RET true if pending dependencies should be reconsidered | |
| */ | |
| static bool _publish_step_outcome(step_record_t *step_ptr, bool started) | |
| { | |
| if (!started) | |
| srun_step_drained(step_ptr); | |
| return step_depend_outcome_update(step_ptr, started); | |
| } | |
| static void _internal_step_complete(step_record_t *step_ptr, int remaining) | |
| { | |
| struct jobacctinfo *jobacct = (struct jobacctinfo *)step_ptr->jobacct; | |
| job_record_t *job_ptr = step_ptr->job_ptr; | |
| bool add_energy = true; | |
| if ((slurm_conf.prolog_flags & PROLOG_FLAG_CONTAIN) && | |
| (step_ptr->step_id.step_id != SLURM_EXTERN_CONT)) | |
| add_energy = false; | |
| if (add_energy && jobacct && job_ptr->tres_alloc_cnt && | |
| (jobacct->energy.consumed_energy != NO_VAL64)) { | |
| if (job_ptr->tres_alloc_cnt[TRES_ARRAY_ENERGY] == NO_VAL64) | |
| job_ptr->tres_alloc_cnt[TRES_ARRAY_ENERGY] = 0; | |
| job_ptr->tres_alloc_cnt[TRES_ARRAY_ENERGY] += | |
| jobacct->energy.consumed_energy; | |
| } | |
| if (IS_JOB_FINISHED(job_ptr) && | |
| job_ptr->tres_alloc_cnt && | |
| (job_ptr->tres_alloc_cnt[TRES_ENERGY] != NO_VAL64) && | |
| (remaining == 1)) { | |
| assoc_mgr_set_job_tres_alloc_str(job_ptr, false); | |
| /* This flag says we have processed the tres alloc including | |
| * energy from all steps, so don't process or handle it again | |
| * with the job. It also tells the slurmdbd plugin to send it | |
| * to the DBD. | |
| */ | |
| job_ptr->bit_flags |= TRES_STR_CALC; | |
| } | |
| /* | |
| * This is where a step irreversibly finishes, however it got here, so | |
| * it is the one place its dependents have to hear about. A pending | |
| * placeholder counts too: it was cancelled or failed before it could | |
| * run, which an afterok dependent must not mistake for success. | |
| */ | |
| if (IS_STEP_PENDING(step_ptr)) { | |
| /* | |
| * A placeholder torn down before it ever launched still has to | |
| * be reported, or a per-step wait on it only ever sees the | |
| * whole-set drain. The push reads state; force it terminal | |
| * first, matching the signal path in _delete_pending_steps(). | |
| */ | |
| _complete_unlaunched_step(step_ptr); | |
| _publish_step_outcome(step_ptr, false); | |
| return; | |
| } | |
| jobacct_storage_g_step_complete(stepmgr_ops->acct_db_conn, step_ptr); | |
| /* | |
| * Derived exit code is the highest exit code of srun steps, so we | |
| * exclude the batch and extern steps. | |
| * | |
| * Sync with _get_derived_ec_update_str() for setting derived_ec on the | |
| * dbd side. | |
| */ | |
| if ((step_ptr->step_id.step_id != SLURM_EXTERN_CONT) && | |
| (step_ptr->step_id.step_id != SLURM_BATCH_SCRIPT) && | |
| ((step_ptr->exit_code == SIG_OOM) || | |
| (step_ptr->exit_code > job_ptr->derived_ec))) | |
| job_ptr->derived_ec = step_ptr->exit_code; | |
| step_ptr->state |= JOB_COMPLETING; | |
| _publish_step_outcome(step_ptr, false); | |
| _step_dealloc_lps(step_ptr); | |
| /* Don't need to set state. Will be destroyed in next steps. */ | |
| /* step_ptr->state = JOB_COMPLETE; */ | |
| } | |
| /* | |
| * Complete and delete a failed step without waking its dependents. | |
| * IN step_ptr - the failed step, or NULL | |
| * RET true if a step was removed | |
| */ | |
| static bool _cleanup_failed_step(step_record_t *step_ptr) | |
| { | |
| job_record_t *job_ptr = NULL; | |
| if (!step_ptr) | |
| return false; | |
| job_ptr = step_ptr->job_ptr; | |
| _internal_step_complete(step_ptr, 0); | |
| delete_step_record(job_ptr, step_ptr); | |
| return true; | |
| } | |
| /* | |
| * Whether a signal cancels a queued (taskless) step. | |
| * IN signal - signal number from the kill request | |
| * RET true if the signal aborts the pending step, false to ignore it | |
| */ | |
| static bool _sig_aborts_pending_step(uint16_t signal) | |
| { | |
| switch (signal) { | |
| case SIGINT: | |
| case SIGTERM: | |
| case SIGKILL: | |
| return true; | |
| default: | |
| return false; | |
| } | |
| } | |
| static int _step_signal(void *object, void *arg) | |
| { | |
| step_record_t *step_ptr = (step_record_t *)object; | |
| step_signal_t *step_signal = (step_signal_t *)arg; | |
| uint16_t signal; | |
| int rc; | |
| if (!(step_signal->flags & KILL_FULL_JOB) && | |
| !find_step_id(step_ptr, &step_signal->step_id)) | |
| return SLURM_SUCCESS; | |
| if (IS_STEP_PENDING(step_ptr)) { | |
| /* | |
| * A terminating signal aborts the queued create (the | |
| * placeholder is reaped in _delete_pending_steps()), | |
| * anything else is a no-op. Nudge the waiting srun | |
| * for synchronous steps. | |
| */ | |
| step_signal->found = true; | |
| if (_sig_aborts_pending_step(step_signal->signal) && | |
| !(step_ptr->flags & SSF_ASYNC)) | |
| srun_step_signal(step_ptr, step_signal->signal); | |
| return SLURM_SUCCESS; | |
| } | |
| if (IS_STEP_CONFIGURING(step_ptr)) { | |
| /* | |
| * Provisional step: created at wake and pushed to srun, but | |
| * the push outcome is not in yet. The push runs without the | |
| * stepmgr lock, so releasing the step's CPUs here could pull | |
| * them out from under a srun that already took the credential. | |
| * Record the requester instead and let the outcome callback act | |
| * on it: a failed push reaps the step, a delivered one finishes | |
| * the cancel as for a running step. | |
| */ | |
| step_signal->found = true; | |
| /* | |
| * Carry the modifiers the running-step arm would have applied, | |
| * so they still take effect if the step goes on to launch. | |
| */ | |
| if (step_signal->flags & KILL_OOM) | |
| step_ptr->exit_code = SIG_OOM; | |
| if (step_signal->flags & KILL_NO_SIG_FAIL) | |
| step_ptr->flags |= SSF_NO_SIG_FAIL; | |
| if (_sig_aborts_pending_step(step_signal->signal)) { | |
| xfree(step_ptr->kill_user); | |
| step_ptr->kill_user = xstrdup(step_signal->kill_user); | |
| srun_step_signal(step_ptr, step_signal->signal); | |
| } | |
| return SLURM_SUCCESS; | |
| } | |
| step_signal->found = true; | |
| signal = step_signal->signal; | |
| /* | |
| * If step_het_comp is NO_VAL means it is a non-het step, so return | |
| * SLURM_ERROR to break out of the list_for_each. | |
| */ | |
| rc = (step_ptr->step_id.step_het_comp == NO_VAL) ? | |
| SLURM_ERROR : SLURM_SUCCESS; | |
| if (step_signal->flags & KILL_OOM) | |
| step_ptr->exit_code = SIG_OOM; | |
| if (step_signal->flags & KILL_NO_SIG_FAIL) { | |
| debug("%s: setting SSF_NO_SIG_FAIL for %pS", | |
| __func__, step_ptr); | |
| step_ptr->flags |= SSF_NO_SIG_FAIL; | |
| } | |
| /* | |
| * If SIG_NODE_FAIL codes through it means we had nodes failed | |
| * so handle that in the select plugin and switch the signal | |
| * to KILL afterwards. | |
| */ | |
| if (signal == SIG_NODE_FAIL) { | |
| if (step_signal->rc_in != SLURM_SUCCESS) | |
| return rc; | |
| signal = SIGKILL; | |
| } | |
| /* save the name of the user that requested the step be cancelled */ | |
| if (signal == SIGKILL) { | |
| xfree(step_ptr->kill_user); | |
| step_ptr->kill_user = xstrdup(step_signal->kill_user); | |
| srun_step_complete(step_ptr); | |
| } | |
| signal_step_tasks(step_ptr, signal, REQUEST_SIGNAL_TASKS); | |
| return rc; | |
| } | |
| static int _step_not_cleaning(void *x, void *arg) | |
| { | |
| step_record_t *step_ptr = (step_record_t *) x; | |
| int *remaining = (int *) arg; | |
| if (IS_STEP_PENDING(step_ptr) && !(step_ptr->flags & SSF_ASYNC)) | |
| srun_step_signal(step_ptr, 0); | |
| _internal_step_complete(step_ptr, *remaining); | |
| (*remaining)--; | |
| return 1; | |
| } | |
| /* | |
| * _finish_step_comp - Finish deallocating and delete a non-pending step. | |
| */ | |
| static int _finish_step_comp(void *x, void *arg) | |
| { | |
| int remaining; | |
| step_record_t *step_ptr = x; | |
| job_record_t *job_ptr = step_ptr->job_ptr; | |
| if (IS_STEP_PENDING(step_ptr) && !(step_ptr->flags & SSF_ASYNC)) | |
| return 0; | |
| remaining = list_count(job_ptr->step_list); | |
| _internal_step_complete(step_ptr, remaining); | |
| delete_step_record(job_ptr, step_ptr); | |
| _wake_pending_steps(job_ptr); | |
| *stepmgr_ops->last_job_update = time(NULL); | |
| return 1; | |
| } | |
| /* | |
| * delete_step_records - Delete step record for specified job_ptr. | |
| * This function is called when a step fails to run to completion. For example, | |
| * when the job is killed due to reaching its time limit or allocated nodes | |
| * go DOWN. | |
| * IN job_ptr - pointer to job table entry to have step records removed | |
| */ | |
| extern void delete_step_records(job_record_t *job_ptr) | |
| { | |
| int remaining; | |
| xassert(job_ptr); | |
| remaining = list_count(job_ptr->step_list); | |
| *stepmgr_ops->last_job_update = time(NULL); | |
| list_delete_all(job_ptr->step_list, _step_not_cleaning, &remaining); | |
| srun_steps_drained(job_ptr); | |
| } | |
| /* | |
| * delete_step_record - delete record for job step for specified job_ptr | |
| * and step_id | |
| * IN job_ptr - pointer to job table entry to have step record removed | |
| * IN step_ptr - pointer to step table entry of the desired job step | |
| */ | |
| void delete_step_record(job_record_t *job_ptr, step_record_t *step_ptr) | |
| { | |
| xassert(job_ptr); | |
| xassert(job_ptr->step_list); | |
| xassert(step_ptr); | |
| *stepmgr_ops->last_job_update = time(NULL); | |
| list_delete_ptr(job_ptr->step_list, step_ptr); | |
| srun_steps_drained(job_ptr); | |
| } | |
| /* | |
| * _dump_step_desc - dump the incoming step initiate request message | |
| * IN step_spec - job step request specification from RPC | |
| */ | |
| static void _dump_step_desc(step_create_req_msg_t *step_spec) | |
| { | |
| uint64_t mem_value = step_spec->pn_min_memory; | |
| char *mem_type = "node"; | |
| if (mem_value & MEM_PER_CPU) { | |
| mem_value &= (~MEM_PER_CPU); | |
| mem_type = "cpu"; | |
| } | |
| log_flag(CPU_FREQ, "StepDesc: %pI cpu_freq_gov=%u cpu_freq_max=%u cpu_freq_min=%u", | |
| &step_spec->step_id, step_spec->cpu_freq_gov, | |
| step_spec->cpu_freq_max, step_spec->cpu_freq_min); | |
| debug3("StepDesc: %ps node_count=%u-%u cpu_count=%u num_tasks=%u", | |
| &step_spec->step_id, step_spec->min_nodes, step_spec->max_nodes, | |
| step_spec->cpu_count, step_spec->num_tasks); | |
| debug3(" cpu_freq_gov=%u cpu_freq_max=%u cpu_freq_min=%u " | |
| "relative=%u task_dist=0x%X plane=%u", | |
| step_spec->cpu_freq_gov, step_spec->cpu_freq_max, | |
| step_spec->cpu_freq_min, step_spec->relative, | |
| step_spec->task_dist, step_spec->plane_size); | |
| debug3(" node_list=%s constraints=%s", | |
| step_spec->node_list, step_spec->features); | |
| debug3(" host=%s port=%u srun_pid=%u name=%s network=%s exclusive=%s", | |
| step_spec->host, step_spec->port, step_spec->srun_pid, | |
| step_spec->name, step_spec->network, | |
| (step_spec->flags & SSF_EXCLUSIVE) ? "yes" : "no"); | |
| debug3(" mem_per_%s=%"PRIu64" resv_port_cnt=%u immediate=%u no_kill=%s", | |
| mem_type, mem_value, step_spec->resv_port_cnt, | |
| step_spec->immediate, | |
| (step_spec->flags & SSF_NO_KILL) ? "yes" : "no"); | |
| debug3(" overcommit=%s time_limit=%u async=%s", | |
| (step_spec->flags & SSF_OVERCOMMIT) ? "yes" : "no", | |
| step_spec->time_limit, | |
| (step_spec->flags & SSF_ASYNC) ? "yes" : "no"); | |
| if (step_spec->cpus_per_tres) | |
| debug3(" CPUs_per_TRES=%s", step_spec->cpus_per_tres); | |
| if (step_spec->mem_per_tres) | |
| debug3(" Mem_per_TRES=%s", step_spec->mem_per_tres); | |
| if (step_spec->tres_bind) | |
| debug3(" TRES_bind=%s", step_spec->tres_bind); | |
| if (step_spec->tres_freq) | |
| debug3(" TRES_freq=%s", step_spec->tres_freq); | |
| if (step_spec->tres_per_step) | |
| debug3(" TRES_per_step=%s", step_spec->tres_per_step); | |
| if (step_spec->tres_per_node) | |
| debug3(" TRES_per_node=%s", step_spec->tres_per_node); | |
| if (step_spec->tres_per_socket) | |
| debug3(" TRES_per_socket=%s", step_spec->tres_per_socket); | |
| if (step_spec->tres_per_task) | |
| debug3(" TRES_per_task=%s", step_spec->tres_per_task); | |
| if (step_spec->runtime || step_spec->container || | |
| step_spec->container_id) | |
| debug3(" Runtime=%s Container=%s ContainerID=%s", | |
| step_spec->runtime, step_spec->container, | |
| step_spec->container_id); | |
| } | |
| static int _delete_pending_steps(void *x, void *arg) | |
| { | |
| step_record_t *step_ptr = x; | |
| step_signal_t *step_signal = arg; | |
| if (IS_STEP_PENDING(step_ptr) && | |
| verify_step_id(&step_ptr->step_id, &step_signal->step_id)) { | |
| /* | |
| * Reap a placeholder only on a terminating signal, | |
| * synchronous or asynchronous. | |
| */ | |
| if (!_sig_aborts_pending_step(step_signal->signal)) | |
| return 0; | |
| step_signal->found = true; | |
| _complete_unlaunched_step(step_ptr); | |
| _publish_step_outcome(step_ptr, false); | |
| return 1; | |
| } | |
| return 0; | |
| } | |
| extern int job_step_signal(slurm_msg_t *msg, slurm_step_id_t *step_id, | |
| uint16_t signal, uint16_t flags, char *kill_user) | |
| { | |
| job_record_t *job_ptr; | |
| int pending_steps_removed = 0; | |
| step_signal_t step_signal = { | |
| .flags = flags, | |
| .found = false, | |
| .kill_user = kill_user ? kill_user : get_msg_username(msg), | |
| .rc_in = SLURM_SUCCESS, | |
| .signal = signal, | |
| .step_id = *step_id, | |
| }; | |
| if (!(job_ptr = stepmgr_ops->find_job(step_id))) { | |
| error("%s: invalid %pI", __func__, step_id); | |
| return ESLURM_INVALID_JOB_ID; | |
| } | |
| if (IS_JOB_FINISHED(job_ptr)) { | |
| step_signal.rc_in = ESLURM_ALREADY_DONE; | |
| if (signal != SIG_NODE_FAIL) | |
| return step_signal.rc_in; | |
| } else if (!IS_JOB_RUNNING(job_ptr)) { | |
| verbose("%s: %pJ is in state %s, cannot signal steps", | |
| __func__, job_ptr, | |
| job_state_string(job_ptr->job_state)); | |
| if (signal != SIG_NODE_FAIL) | |
| return ESLURM_TRANSITION_STATE_NO_UPDATE; | |
| } | |
| list_for_each(job_ptr->step_list, _step_signal, &step_signal); | |
| pending_steps_removed = | |
| list_delete_all(job_ptr->step_list, _delete_pending_steps, | |
| &step_signal); | |
| if ((pending_steps_removed > 0) && IS_JOB_RUNNING(job_ptr)) | |
| _wake_pending_steps(job_ptr); | |
| else | |
| srun_steps_drained(job_ptr); | |
| if (!step_signal.found && running_in_slurmctld() && | |
| (job_ptr->bit_flags & STEPMGR_ENABLED)) { | |
| agent_arg_t *agent_args = NULL; | |
| job_step_kill_msg_t *kill_msg = NULL; | |
| node_record_t *node_ptr; | |
| kill_msg = xmalloc(sizeof(*kill_msg)); | |
| kill_msg->signal = signal; | |
| kill_msg->flags = flags; | |
| kill_msg->kill_user = xstrdup(step_signal.kill_user); | |
| kill_msg->step_id = *step_id; | |
| agent_args = xmalloc(sizeof(agent_arg_t)); | |
| agent_args->msg_type = REQUEST_CANCEL_JOB_STEP; | |
| agent_args->retry = 1; | |
| agent_args->hostlist = hostlist_create(job_ptr->batch_host); | |
| agent_args->node_count = 1; | |
| if ((node_ptr = find_node_record(job_ptr->batch_host))) | |
| agent_args->protocol_version = | |
| node_ptr->protocol_version; | |
| agent_args->msg_args = kill_msg; | |
| set_agent_arg_r_uid(agent_args, slurm_conf.slurmd_user_id); | |
| stepmgr_ops->agent_queue_request(agent_args); | |
| step_signal.found = true; | |
| step_signal.rc_in = SLURM_SUCCESS; | |
| } | |
| if (!step_signal.found) { | |
| info("%s: %pJ StepId=%u not found", | |
| __func__, job_ptr, step_id->step_id); | |
| return ESLURM_INVALID_JOB_ID; | |
| } | |
| return step_signal.rc_in; | |
| } | |
| /* slurmstepd is never sent SlurmctldParameters, so use the job record's copy */ | |
| static void _set_resolve_hostlist(job_record_t *job_ptr, | |
| agent_arg_t *agent_args) | |
| { | |
| if (job_ptr->resolve_nodes) | |
| agent_args->resolve_hostlist = | |
| hostlist_create(job_ptr->resolve_nodes); | |
| } | |
| /* | |
| * signal_step_tasks - send specific signal to specific job step | |
| * IN step_ptr - step record pointer | |
| * IN signal - signal to send | |
| * IN msg_type - message type to send | |
| */ | |
| void signal_step_tasks(step_record_t *step_ptr, uint16_t signal, | |
| slurm_msg_type_t msg_type) | |
| { | |
| node_record_t *node_ptr; | |
| static bool cloud_dns = false; | |
| static time_t last_update = 0; | |
| job_record_t *job_ptr = step_ptr->job_ptr; | |
| signal_tasks_msg_t *signal_tasks_msg; | |
| agent_arg_t *agent_args = NULL; | |
| xassert(step_ptr); | |
| agent_args = xmalloc(sizeof(agent_arg_t)); | |
| agent_args->msg_type = msg_type; | |
| agent_args->retry = 1; | |
| agent_args->hostlist = hostlist_create(NULL); | |
| signal_tasks_msg = xmalloc(sizeof(signal_tasks_msg_t)); | |
| memcpy(&signal_tasks_msg->step_id, &step_ptr->step_id, | |
| sizeof(signal_tasks_msg->step_id)); | |
| signal_tasks_msg->signal = signal; | |
| if (step_ptr->flags & SSF_NO_SIG_FAIL) | |
| signal_tasks_msg->flags |= KILL_NO_SIG_FAIL; | |
| log_flag(STEPS, "%s: queueing signal %d with flags=0x%x for %pS", | |
| __func__, signal, signal_tasks_msg->flags, step_ptr); | |
| if (last_update != slurm_conf.last_update) { | |
| if (xstrcasestr(slurm_conf.slurmctld_params, "cloud_dns")) | |
| cloud_dns = true; | |
| else | |
| cloud_dns = false; | |
| last_update = slurm_conf.last_update; | |
| } | |
| agent_args->protocol_version = SLURM_PROTOCOL_VERSION; | |
| for (int i = 0; | |
| (node_ptr = next_node_bitmap(step_ptr->step_node_bitmap, &i)); | |
| i++) { | |
| if (agent_args->protocol_version > node_ptr->protocol_version) | |
| agent_args->protocol_version = | |
| node_ptr->protocol_version; | |
| hostlist_push_host(agent_args->hostlist, node_ptr->name); | |
| agent_args->node_count++; | |
| if (PACK_FANOUT_ADDRS(node_ptr)) | |
| agent_args->msg_flags |= SLURM_PACK_ADDRS; | |
| } | |
| if (agent_args->node_count == 0) { | |
| xfree(signal_tasks_msg); | |
| hostlist_destroy(agent_args->hostlist); | |
| xfree(agent_args); | |
| return; | |
| } | |
| _set_resolve_hostlist(job_ptr, agent_args); | |
| agent_args->msg_args = signal_tasks_msg; | |
| set_agent_arg_r_uid(agent_args, SLURM_AUTH_UID_ANY); | |
| stepmgr_ops->agent_queue_request(agent_args); | |
| } | |
| /* | |
| * signal_step_tasks_on_node - send specific signal to specific job step | |
| * on a specific node. | |
| * IN node_name - name of node on which to signal tasks | |
| * IN step_ptr - step record pointer | |
| * IN signal - signal to send | |
| * IN msg_type - message type to send | |
| */ | |
| void signal_step_tasks_on_node(char* node_name, step_record_t *step_ptr, | |
| uint16_t signal, slurm_msg_type_t msg_type) | |
| { | |
| signal_tasks_msg_t *signal_tasks_msg; | |
| agent_arg_t *agent_args = NULL; | |
| node_record_t *node_ptr; | |
| xassert(step_ptr); | |
| agent_args = xmalloc(sizeof(agent_arg_t)); | |
| agent_args->msg_type = msg_type; | |
| agent_args->retry = 1; | |
| if ((node_ptr = find_node_record(node_name))) | |
| agent_args->protocol_version = node_ptr->protocol_version; | |
| agent_args->node_count++; | |
| agent_args->hostlist = hostlist_create(node_name); | |
| if (!agent_args->hostlist) | |
| fatal("Invalid node_name: %s", node_name); | |
| signal_tasks_msg = xmalloc(sizeof(signal_tasks_msg_t)); | |
| memcpy(&signal_tasks_msg->step_id, &step_ptr->step_id, | |
| sizeof(signal_tasks_msg->step_id)); | |
| signal_tasks_msg->signal = signal; | |
| agent_args->msg_args = signal_tasks_msg; | |
| set_agent_arg_r_uid(agent_args, SLURM_AUTH_UID_ANY); | |
| stepmgr_ops->agent_queue_request(agent_args); | |
| } | |
| typedef struct { | |
| int config_start_count; | |
| int start_count; | |
| time_t max_age; | |
| bool push_failed; | |
| bool made_progress; | |
| } wake_steps_args_t; | |
| /* | |
| * Find the job a step create request is for. | |
| * | |
| * srun resolves --jobid=<array_job_id>_<task_id> to the task's own job id, but | |
| * a het step still carries the task id. A task that runs under its own job id | |
| * is not found by its array job id, so also accept the job found by its own id | |
| * when it is that task. | |
| */ | |
| static job_record_t *_find_step_req_job(slurm_step_id_t *step_id, | |
| uint32_t array_task_id) | |
| { | |
| job_record_t *job_ptr; | |
| if (array_task_id == NO_VAL) | |
| return stepmgr_ops->find_job(step_id); | |
| if ((job_ptr = stepmgr_ops->find_job_array_rec(step_id->job_id, | |
| array_task_id))) | |
| return job_ptr; | |
| if ((job_ptr = stepmgr_ops->find_job(step_id)) && | |
| (job_ptr->array_task_id == array_task_id)) | |
| return job_ptr; | |
| return NULL; | |
| } | |
| /* | |
| * Copy a retained step create request through a pack/unpack round trip, so | |
| * the mutations a wake attempt makes (node pick, task count, port count) | |
| * never reach the request retained on the placeholder for the next attempt. | |
| * IN step_req - the retained request to copy | |
| * IN protocol_version - version to pack at (the srun's negotiated version) | |
| * RET the copy, or NULL on pack/unpack failure | |
| */ | |
| static step_create_req_msg_t *_copy_step_req(step_create_req_msg_t *step_req, | |
| uint16_t protocol_version) | |
| { | |
| slurm_msg_t msg; | |
| buf_t *buffer = init_buf(0); | |
| step_create_req_msg_t *copy = NULL; | |
| slurm_msg_t_init(&msg); | |
| msg.protocol_version = protocol_version; | |
| msg.data = step_req; | |
| pack_job_step_create_request_msg(&msg, buffer); | |
| set_buf_offset(buffer, 0); | |
| msg.data = NULL; | |
| if (!unpack_job_step_create_request_msg(&msg, buffer)) | |
| copy = msg.data; | |
| free_buf(buffer); | |
| return copy; | |
| } | |
| /* | |
| * Create a pending step's real step_record at wake time and build its | |
| * step-create response (the shared spine of the async and sync wake arms). | |
| * On a terminal error before the replacement step commits, logs and records | |
| * accounting for placeholder that reserved real id at build (only ones that | |
| * got step_start). | |
| * IN job_ptr - owning job | |
| * IN pend_step_ptr - the pending placeholder (carries step_req) | |
| * OUT new_step_ptr - the created step_record on success | |
| * OUT step_resp - the built response (cred, layout, ports) on success | |
| * OUT dependency_reconsider - pending dependencies should be reconsidered | |
| * RET SLURM_SUCCESS, ESLURM_STEP_QUEUED (still busy, keep pending), or a | |
| * terminal error (already logged/cleaned) | |
| */ | |
| static int _create_step_at_wake(job_record_t *job_ptr, | |
| step_record_t *pend_step_ptr, | |
| step_record_t **new_step_ptr, | |
| step_create_resp_msg_t **step_resp, | |
| bool *dependency_reconsider) | |
| { | |
| step_create_req_msg_t *step_req = NULL; | |
| step_create_resp_msg_t *resp = NULL; | |
| slurm_cred_t *slurm_cred = NULL; | |
| char *err_msg = NULL; | |
| int rc = EINVAL; | |
| *dependency_reconsider = false; | |
| step_req = _copy_step_req(pend_step_ptr->step_req, | |
| pend_step_ptr->start_protocol_ver); | |
| if (!step_req) { | |
| error("Could not copy the retained step request for a wake attempt"); | |
| rc = SLURM_ERROR; | |
| } else { | |
| slurm_msg_t msg = { | |
| .auth = SLURM_MSG_AUTH_INIT_FROM_JOB(job_ptr), | |
| .protocol_version = pend_step_ptr->start_protocol_ver, | |
| }; | |
| step_req->immediate = 1; | |
| rc = _step_create(&msg, job_ptr, step_req, new_step_ptr, | |
| pend_step_ptr, dependency_reconsider, | |
| &err_msg); | |
| xfree(err_msg); | |
| } | |
| if (rc == ESLURM_STEP_QUEUED) { | |
| debug2("Resources busy, step still queued"); | |
| slurm_free_step_create_req_msg(step_req); | |
| return rc; | |
| } else if (rc) { | |
| if (rc == ESLURM_ALREADY_DONE) | |
| debug("%s: job for pending %ps is ending, dropping", | |
| __func__, &pend_step_ptr->step_id); | |
| else | |
| error("Could not create step at wake: %s", | |
| slurm_strerror(rc)); | |
| /* | |
| * A placeholder that reserved a real id (async, or sync with a | |
| * >= 26.11 srun) got a jobacct step_start at build; balance it | |
| * here. | |
| */ | |
| _complete_unlaunched_step(pend_step_ptr); | |
| *dependency_reconsider |= | |
| _publish_step_outcome(pend_step_ptr, false); | |
| slurm_free_step_create_req_msg(step_req); | |
| return rc; | |
| } | |
| rc = _make_step_cred(*new_step_ptr, &slurm_cred, | |
| (*new_step_ptr)->start_protocol_ver); | |
| if (rc) { | |
| error("Could not create cred for step at wake: %s", | |
| slurm_strerror(rc)); | |
| *dependency_reconsider |= _cleanup_failed_step(*new_step_ptr); | |
| slurm_free_step_create_req_msg(step_req); | |
| return rc; | |
| } | |
| xassert(!(*new_step_ptr)->step_dependency); | |
| (*new_step_ptr)->step_dependency = pend_step_ptr->step_dependency; | |
| pend_step_ptr->step_dependency = NULL; | |
| resp = xmalloc(sizeof(*resp)); | |
| /* | |
| * Async steps carry launch_params; a sync step does not, so fall back | |
| * to the job's default bind as the regular create reply does. | |
| */ | |
| if (step_req->launch_params) | |
| resp->def_cpu_bind_type = | |
| step_req->launch_params->cpu_bind_type; | |
| else if (job_ptr->details && | |
| (job_ptr->details->cpu_bind_type != NO_VAL16)) | |
| resp->def_cpu_bind_type = job_ptr->details->cpu_bind_type; | |
| resp->resv_ports = xstrdup((*new_step_ptr)->resv_ports); | |
| resp->state = JOB_RUNNING; | |
| resp->step_id = (*new_step_ptr)->step_id; | |
| resp->step_layout = | |
| slurm_step_layout_copy((*new_step_ptr)->step_layout); | |
| resp->stepmgr = xstrdup(job_ptr->batch_host); | |
| resp->cred = slurm_cred; | |
| resp->use_protocol_ver = (*new_step_ptr)->start_protocol_ver; | |
| step_req->use_protocol_ver = (*new_step_ptr)->start_protocol_ver; | |
| pend_step_ptr->step_req->use_protocol_ver = step_req->use_protocol_ver; | |
| *step_resp = resp; | |
| slurm_free_step_create_req_msg(step_req); | |
| return SLURM_SUCCESS; | |
| } | |
| /* | |
| * Launch a created async step in-process (the async delivery tail) and free | |
| * the retained step_req on success. | |
| * IN job_ptr - owning job | |
| * IN/OUT pend_step_ptr - the pending placeholder (step_req cleared on success) | |
| * IN new_step_ptr - the created step_record | |
| * IN step_resp - the built response (consumed by the ctx) | |
| * RET SLURM_SUCCESS or launch error; on error the step still needs | |
| * completing and freeing by the caller | |
| */ | |
| static int _launch_in_stepmgr(job_record_t *job_ptr, | |
| step_record_t *pend_step_ptr, | |
| step_record_t *new_step_ptr, | |
| step_create_resp_msg_t *step_resp) | |
| { | |
| step_create_req_msg_t *step_req = pend_step_ptr->step_req; | |
| slurm_step_ctx_t *ctx = NULL; | |
| int rc = EINVAL; | |
| /* | |
| * Set to "none" when not set, to prevent | |
| * mpi_g_client_init() setting to MpiDefault. | |
| */ | |
| if (!step_req->launch_params->mpi_plugin_name) | |
| step_req->launch_params->mpi_plugin_name = xstrdup("none"); | |
| ctx = _step_ctx_create_stepmgr(step_req, step_resp, new_step_ptr); | |
| rc = slurm_step_launch(ctx, step_req->launch_params, NULL); | |
| ctx->step_req = NULL; | |
| step_ctx_destroy(ctx); | |
| if (rc) { | |
| error("Could not launch async step: %s", slurm_strerror(rc)); | |
| return rc; | |
| } | |
| /* | |
| * step_req was retained on pend_step_ptr across _build_pending_step | |
| * until launch. Free it now and clear the alias so free_step_record() | |
| * does not double-free it. | |
| */ | |
| slurm_free_step_create_req_msg(step_req); | |
| pend_step_ptr->step_req = NULL; | |
| return SLURM_SUCCESS; | |
| } | |
| /* | |
| * True if this pending step is (re)created under the lock at wake and its | |
| * result delivered (create-at-wake), rather than poked to re-send. Async | |
| * always is. A sync step is only in the batch-node stepmgr for a >= 26.11 | |
| * srun that can consume the push; that is exactly when _build_pending_step() | |
| * retained step_req and captured the srun's version on the placeholder. | |
| * IN step_ptr - the pending placeholder | |
| * RET true if the step is (re)created at wake, false if it is poked to re-send | |
| */ | |
| static bool _step_uses_create_at_wake(step_record_t *step_ptr) | |
| { | |
| if (step_ptr->flags & SSF_ASYNC) | |
| return true; | |
| return (!running_in_slurmctld() && step_ptr->step_req && | |
| (step_ptr->start_protocol_ver >= SLURM_26_11_PROTOCOL_VERSION)); | |
| } | |
| /* | |
| * Finish a pending step whose dependencies can never be satisfied. It never | |
| * ran, so it is recorded as failed rather than left to derive a status from an | |
| * exit code it does not have, and its own dependents hear about it in turn. | |
| * Its record is deleted by the caller as the wake walk's current entry. | |
| * IN job_ptr - owning job | |
| * IN step_ptr - the placeholder to finish | |
| * RET true if pending dependencies should be reconsidered | |
| */ | |
| static bool _fail_dep_step(job_record_t *job_ptr, step_record_t *step_ptr) | |
| { | |
| info("Failing %pS; its step dependency can never be satisfied", | |
| step_ptr); | |
| /* Tell the waiting srun why it will never get its step. */ | |
| if (!(step_ptr->flags & SSF_ASYNC)) | |
| srun_step_alloc_failed(step_ptr, ESLURM_STEP_DEP_FAILED); | |
| _fail_unlaunched_step(step_ptr); | |
| return _publish_step_outcome(step_ptr, false); | |
| } | |
| /* | |
| * Wake one pending step during list_delete_all(). | |
| * IN x - step record to test | |
| * IN/OUT arg - wake arguments shared across this wake drain | |
| * RET 1 if placeholder should be retired, 0 to keep it | |
| */ | |
| static int _wake_steps(void *x, void *arg) | |
| { | |
| step_record_t *pend_step_ptr = x; | |
| wake_steps_args_t *args = arg; | |
| job_record_t *job_ptr = NULL; | |
| step_record_t *new_step_ptr = NULL; | |
| step_create_resp_msg_t *step_resp = NULL; | |
| step_create_req_msg_t *step_req = NULL; | |
| bool dependency_reconsider = false; | |
| bool is_async = false; | |
| int rc = EINVAL; | |
| if (!IS_STEP_PENDING(pend_step_ptr)) | |
| return 0; | |
| /* | |
| * Dependency waits spend no retry budget; impossible steps fail now. | |
| * Every outcome change wakes this walk, so the reason recorded here | |
| * stays current for step info. | |
| */ | |
| pend_step_ptr->state_reason = | |
| step_depend_reduce_list(pend_step_ptr->step_depend_list, | |
| pend_step_ptr->step_depend_or); | |
| switch (pend_step_ptr->state_reason) { | |
| case WAIT_DEPENDENCY: | |
| return 0; | |
| case WAIT_DEP_INVALID: | |
| args->made_progress |= | |
| _fail_dep_step(pend_step_ptr->job_ptr, pend_step_ptr); | |
| return 1; | |
| default: | |
| /* Dependencies are met, so only resources hold it back. */ | |
| pend_step_ptr->state_reason = WAIT_RESOURCES; | |
| break; | |
| } | |
| if (!((args->start_count < args->config_start_count) || | |
| (pend_step_ptr->time_last_active <= args->max_age))) | |
| return 0; | |
| /* Controller-managed step, or an old srun: poke + re-send. */ | |
| if (!_step_uses_create_at_wake(pend_step_ptr)) { | |
| /* | |
| * Close the accounting row a real-id placeholder got at | |
| * build: the re-send records the step for real, and a | |
| * re-send that never comes must not leave a PENDING row | |
| * open forever. | |
| */ | |
| _complete_unlaunched_step(pend_step_ptr); | |
| srun_step_signal(pend_step_ptr, 0); | |
| args->start_count++; | |
| return 1; | |
| } | |
| is_async = (pend_step_ptr->flags & SSF_ASYNC); | |
| step_req = pend_step_ptr->step_req; | |
| debug2("Attempting create-at-wake of pending %s step", | |
| is_async ? "async" : "sync"); | |
| _dump_step_desc(step_req); | |
| job_ptr = _find_step_req_job(&pend_step_ptr->step_id, | |
| step_req->array_task_id); | |
| /* | |
| * We hold the lock so it shouldn't be possible for a job_ptr to be | |
| * deleted out from under us, but double check just in case. | |
| */ | |
| if (!job_ptr) { | |
| debug("%s: job for pending step %ps no longer exists, dropping", | |
| __func__, &pend_step_ptr->step_id); | |
| _complete_unlaunched_step(pend_step_ptr); | |
| args->made_progress |= | |
| _publish_step_outcome(pend_step_ptr, false); | |
| return 1; | |
| } | |
| rc = _create_step_at_wake(job_ptr, pend_step_ptr, &new_step_ptr, | |
| &step_resp, &dependency_reconsider); | |
| args->made_progress |= dependency_reconsider; | |
| /* | |
| * Budget only an attempt that could not launch: the pick ran but the | |
| * resources were still busy. A launch consumes no budget, so if | |
| * every pending step fits, every pending step starts. A terminal | |
| * failure is not budgeted either: the step is reaped and its srun | |
| * must be told without waiting out a full budget. | |
| */ | |
| if (rc == ESLURM_STEP_QUEUED) { | |
| /* | |
| * Count the attempt as activity so step_retry_time keeps | |
| * limiting retries; otherwise every long-queued sync step is | |
| * retried on every wake. | |
| */ | |
| pend_step_ptr->time_last_active = time(NULL); | |
| args->start_count++; | |
| return 0; /* still busy: keep pending */ | |
| } | |
| if (rc) { | |
| /* terminal: unblock a waiting sync srun so it stops waiting */ | |
| if (!is_async) | |
| srun_step_alloc_failed(pend_step_ptr, rc); | |
| return 1; /* error already logged + accounted in the spine */ | |
| } | |
| if (is_async) { | |
| rc = _launch_in_stepmgr(job_ptr, pend_step_ptr, new_step_ptr, | |
| step_resp); | |
| if (rc) | |
| args->made_progress |= | |
| _cleanup_failed_step(new_step_ptr); | |
| return 1; | |
| } | |
| /* | |
| * Hand the created step to the waiting srun, which drives the | |
| * launch. Mark it CONFIGURING until the push is delivered (a | |
| * failed push reaps it). _step_create() copied everything it | |
| * needs out of step_req, so free it and clear the alias: that | |
| * way free_step_record(), run when the placeholder unlinks | |
| * below, does not double-free it. | |
| */ | |
| new_step_ptr->state |= JOB_CONFIGURING; | |
| if (!srun_step_alloc(new_step_ptr, step_resp)) | |
| args->push_failed = true; | |
| slurm_free_step_create_req_msg(pend_step_ptr->step_req); | |
| pend_step_ptr->step_req = NULL; | |
| return 1; | |
| } | |
| /* | |
| * Reap a step created at wake that had no srun address to push to, so nothing | |
| * will ever launch it. Deferred until the wake walk finishes: the reap | |
| * mutates the same step_list. | |
| * IN x - step record to test | |
| * IN arg - unused | |
| * RET 1 if the step was completed and should be freed, 0 to keep it | |
| */ | |
| static int _reap_unpushed_step(void *x, void *arg) | |
| { | |
| step_record_t *step_ptr = x; | |
| if (!IS_STEP_CONFIGURING(step_ptr)) | |
| return 0; | |
| if (step_ptr->port && step_ptr->host && step_ptr->host[0]) | |
| return 0; | |
| error("Reaping unlaunched %pS; no srun to push its allocation to", | |
| step_ptr); | |
| /* | |
| * Never launched: complete it as such and release its CPUs/ports/cred | |
| * directly; the full step-completion path is for steps that ran. | |
| */ | |
| _complete_unlaunched_step(step_ptr); | |
| _publish_step_outcome(step_ptr, false); | |
| _step_dealloc_lps(step_ptr); | |
| return 1; | |
| } | |
| /* | |
| * Reconsider pending steps after an event, outside any step-list traversal. | |
| * IN job_ptr - the job whose pending steps should be reconsidered | |
| */ | |
| static void _wake_pending_steps(job_record_t *job_ptr) | |
| { | |
| static int config_start_count = -1, config_max_age = -1; | |
| wake_steps_args_t args = { 0 }; | |
| if (!IS_JOB_RUNNING(job_ptr)) | |
| return; | |
| if (!job_ptr->step_list) | |
| return; | |
| if (config_start_count == -1) { | |
| char *tmp_ptr = NULL; | |
| long int param = LONG_MIN; | |
| config_start_count = 8; | |
| config_max_age = 60; | |
| if ((tmp_ptr = xstrcasestr(slurm_conf.sched_params, | |
| "step_retry_count="))) { | |
| param = strtol(tmp_ptr + 17, NULL, 10); | |
| if ((param >= 1) && (param != LONG_MIN) && | |
| (param != LONG_MAX)) | |
| config_start_count = param; | |
| } | |
| if ((tmp_ptr = xstrcasestr(slurm_conf.sched_params, | |
| "step_retry_time="))) { | |
| param = strtol(tmp_ptr + 16, NULL, 10); | |
| if ((param >= 1) && (param != LONG_MIN) && | |
| (param != LONG_MAX)) | |
| config_max_age = param; | |
| } | |
| } | |
| args.max_age = time(NULL) - config_max_age; | |
| /* We do not know which steps can use currently available resources. | |
| * Try to start a bit more based upon step sizes. Effectiveness | |
| * varies with step sizes, constraints and order. */ | |
| args.config_start_count = config_start_count; | |
| do { | |
| args.made_progress = false; | |
| args.push_failed = false; | |
| list_delete_all(job_ptr->step_list, _wake_steps, &args); | |
| if (args.push_failed && | |
| (list_delete_all(job_ptr->step_list, _reap_unpushed_step, | |
| NULL) > 0)) | |
| args.made_progress = true; | |
| } while (args.made_progress && IS_JOB_RUNNING(job_ptr)); | |
| /* Tell whole-job subscribers once drain has settled. */ | |
| srun_steps_drained(job_ptr); | |
| } | |
| /* | |
| * stepmgr_step_alloc_delivered - clear the provisional mark on a created-at- | |
| * wake step once its push to srun is delivered, so the cancel and | |
| * push-failure paths leave the now-launching step alone. A cancel that | |
| * landed while the push was in flight is finished here as for a running | |
| * step. The signal to srun may arrive after the push, so srun may have | |
| * launched with the delivered credential; keep the step's resources until | |
| * its tasks end or srun reports that it dropped the push. A no-op for any | |
| * other step. | |
| * IN step_id - the delivered step's id (caller holds the step lock) | |
| */ | |
| extern void stepmgr_step_alloc_delivered(slurm_step_id_t *step_id) | |
| { | |
| job_record_t *job_ptr = NULL; | |
| step_record_t *step_ptr = NULL; | |
| if (!(job_ptr = stepmgr_ops->find_job(step_id))) | |
| return; | |
| if (!(step_ptr = find_step_record(job_ptr, step_id))) | |
| return; | |
| step_ptr->state &= ~JOB_CONFIGURING; | |
| if (step_ptr->kill_user) { | |
| srun_step_complete(step_ptr); | |
| signal_step_tasks(step_ptr, SIGKILL, REQUEST_SIGNAL_TASKS); | |
| } | |
| } | |
| /* | |
| * Reap a provisional step whose push to srun could not be delivered. | |
| * IN x - step record to test | |
| * IN arg - slurm_step_id_t of the step whose push failed | |
| * RET 1 if the step was completed and should be freed, 0 to keep it | |
| */ | |
| static int _reap_failed_push_step(void *x, void *arg) | |
| { | |
| step_record_t *step_ptr = x; | |
| slurm_step_id_t *step_id = arg; | |
| if (!IS_STEP_CONFIGURING(step_ptr)) | |
| return 0; | |
| if (!verify_step_id(&step_ptr->step_id, step_id)) | |
| return 0; | |
| /* | |
| * The push could not be delivered: srun's listener is gone, so it will | |
| * never launch this provisional step. Complete it as never-launched | |
| * and release its CPUs/ports/cred directly; returning 1 lets | |
| * list_delete_all() free the record. | |
| */ | |
| info("Reaping unlaunched %pS; push to srun failed", step_ptr); | |
| _complete_unlaunched_step(step_ptr); | |
| _publish_step_outcome(step_ptr, false); | |
| _step_dealloc_lps(step_ptr); | |
| return 1; | |
| } | |
| /* | |
| * stepmgr_step_alloc_failed - reap a provisional create-at-wake step when its | |
| * push to srun could not be delivered, then wake any steps pending on the | |
| * freed resources. A no-op unless the step still exists and is still | |
| * provisional (srun may have re-sent and launched). | |
| * IN step_id - the pushed step's id (caller holds the step lock) | |
| */ | |
| extern void stepmgr_step_alloc_failed(slurm_step_id_t *step_id) | |
| { | |
| job_record_t *job_ptr = NULL; | |
| if (!(job_ptr = stepmgr_ops->find_job(step_id))) | |
| return; | |
| if (!list_delete_all(job_ptr->step_list, _reap_failed_push_step, | |
| step_id)) | |
| return; | |
| *stepmgr_ops->last_job_update = time(NULL); | |
| _wake_pending_steps(job_ptr); | |
| /* | |
| * _wake_pending_steps() tells the steps-drained subscribers, but only | |
| * while the job still runs. Reaping the last step of a job that does | |
| * not would otherwise leave them waiting. | |
| */ | |
| if (!IS_JOB_RUNNING(job_ptr)) | |
| srun_steps_drained(job_ptr); | |
| } | |
| /* | |
| * Set node_inx to the next node in the order map | |
| * | |
| * IN/OUT cur_pos - current position in order_map | |
| * IN/OUT check_cnt - how many nodes returned so far | |
| * IN node_cnt - max nodes to return | |
| * IN nodes_bitmap - nodes available | |
| * IN start_pos - starting position in order_map | |
| * IN order_map - array of (node_inx, node_rank) for all job nodes | |
| * IN order_cnt - number of entries in order_map | |
| * OUT node_inx - the selected node index | |
| * RET SLURM_SUCCESS or SLURM_ERROR when exhausted | |
| */ | |
| static int _next_node_inx(int *cur_pos, int *check_cnt, int node_cnt, | |
| bitstr_t *nodes_bitmap, int start_pos, | |
| node_rank_order_t *order_map, int order_cnt, | |
| int *node_inx) | |
| { | |
| int iter_cnt = 0; | |
| xassert(cur_pos); | |
| xassert(check_cnt); | |
| xassert(nodes_bitmap); | |
| xassert(order_map); | |
| xassert(order_cnt > 0); | |
| xassert(node_inx); | |
| if (*check_cnt >= node_cnt) | |
| return SLURM_ERROR; | |
| if (*check_cnt == 0) { | |
| *cur_pos = start_pos; | |
| } else { | |
| *cur_pos = (*cur_pos + 1) % order_cnt; | |
| if (*cur_pos == start_pos) | |
| return SLURM_ERROR; | |
| } | |
| while (iter_cnt < order_cnt) { | |
| int candidate = order_map[*cur_pos].node_inx; | |
| if (bit_test(nodes_bitmap, candidate)) { | |
| *node_inx = candidate; | |
| (*check_cnt)++; | |
| return SLURM_SUCCESS; | |
| } | |
| *cur_pos = (*cur_pos + 1) % order_cnt; | |
| if (*cur_pos == start_pos) | |
| break; | |
| iter_cnt++; | |
| } | |
| return SLURM_ERROR; | |
| } | |
| /* | |
| * Advance past the current topology unit in order_map, returning the index of | |
| * the first entry whose node_rank differs from order_map[pos]. | |
| */ | |
| static int _next_rank_start(node_rank_order_t *order_map, int order_cnt, | |
| int pos) | |
| { | |
| uint32_t cur_unit = order_map[pos].node_rank >> TOPO_RANK_ID_SHIFT; | |
| int next = (pos + 1) % order_cnt; | |
| /* Sorted array: if first and last share the same unit. */ | |
| if ((order_map[0].node_rank >> TOPO_RANK_ID_SHIFT) == | |
| (order_map[order_cnt - 1].node_rank >> TOPO_RANK_ID_SHIFT)) | |
| return next; | |
| while (next != pos) { | |
| if ((order_map[next].node_rank >> TOPO_RANK_ID_SHIFT) != | |
| cur_unit) | |
| return next; | |
| next = (next + 1) % order_cnt; | |
| } | |
| /* It should never happen with sorted order_map */ | |
| return (pos + 1) % order_cnt; | |
| } | |
| /* Pick nodes to be allocated to a job step. If a CPU count is also specified, | |
| * then select nodes with a sufficient CPU count. | |
| * IN job_ptr - job to contain step allocation | |
| * IN node_bitmap - nodes available (IN) | |
| * IN node_cnt - step node count specification | |
| * IN cpu_cnt - step CPU count specification | |
| * IN usable_cpu_cnt - count of usable CPUs on each node in node_bitmap | |
| * RET nodes selected for use | |
| */ | |
| static bitstr_t *_pick_step_nodes_cpus(job_record_t *job_ptr, | |
| bitstr_t *nodes_bitmap, int node_cnt, | |
| int cpu_cnt, uint32_t *usable_cpu_cnt) | |
| { | |
| bitstr_t *picked_node_bitmap = NULL; | |
| int *usable_cpu_array; | |
| int cpu_target; /* Target number of CPUs per allocated node */ | |
| int rem_nodes, rem_cpus, save_rem_nodes, save_rem_cpus; | |
| int i, pos = 0, check_cnt, node_inx; | |
| int start_pos; | |
| node_rank_order_t *order_map = job_ptr->job_resrcs->order_map; | |
| int order_cnt = job_ptr->job_resrcs->nhosts; | |
| xassert(node_cnt > 0); | |
| xassert(nodes_bitmap); | |
| xassert(usable_cpu_cnt); | |
| xassert(order_map); | |
| xassert(order_cnt > 0); | |
| picked_node_bitmap = bit_alloc(node_record_count); | |
| start_pos = job_ptr->job_resrcs->next_step_node_inx; | |
| if (start_pos >= order_cnt) | |
| start_pos = 0; | |
| cpu_target = ROUNDUP(cpu_cnt, node_cnt); | |
| if (cpu_target > 1024) | |
| info("%s: high cpu_target (%d)", __func__, cpu_target); | |
| if ((cpu_cnt <= node_cnt) || (cpu_target > 1024)) { | |
| check_cnt = 0; | |
| while (_next_node_inx(&pos, &check_cnt, node_cnt, nodes_bitmap, | |
| start_pos, order_map, order_cnt, | |
| &node_inx) == SLURM_SUCCESS) | |
| bit_set(picked_node_bitmap, node_inx); | |
| job_ptr->job_resrcs->next_step_node_inx = | |
| _next_rank_start(order_map, order_cnt, pos); | |
| return picked_node_bitmap; | |
| } | |
| /* Need to satisfy both a node count and a cpu count */ | |
| usable_cpu_array = xcalloc(cpu_target, sizeof(int)); | |
| rem_nodes = node_cnt; | |
| rem_cpus = cpu_cnt; | |
| check_cnt = 0; | |
| while (_next_node_inx(&pos, &check_cnt, order_cnt, nodes_bitmap, | |
| start_pos, order_map, order_cnt, | |
| &node_inx) == SLURM_SUCCESS) { | |
| if (usable_cpu_cnt[node_inx] < cpu_target) { | |
| usable_cpu_array[usable_cpu_cnt[node_inx]]++; | |
| continue; | |
| } | |
| bit_set(picked_node_bitmap, node_inx); | |
| rem_cpus -= usable_cpu_cnt[node_inx]; | |
| rem_nodes--; | |
| if ((rem_cpus <= 0) && (rem_nodes <= 0)) { | |
| /* Satisfied request */ | |
| job_ptr->job_resrcs->next_step_node_inx = | |
| _next_rank_start(order_map, order_cnt, pos); | |
| xfree(usable_cpu_array); | |
| return picked_node_bitmap; | |
| } | |
| if (rem_nodes == 0) { /* Reached node limit, not CPU limit */ | |
| xfree(usable_cpu_array); | |
| FREE_NULL_BITMAP(picked_node_bitmap); | |
| return NULL; | |
| } | |
| } | |
| if (!picked_node_bitmap) { | |
| xfree(usable_cpu_array); | |
| return NULL; | |
| } | |
| /* Need more resources. Determine what CPU counts per node to use */ | |
| save_rem_nodes = rem_nodes; | |
| save_rem_cpus = rem_cpus; | |
| usable_cpu_array[0] = 0; | |
| for (i = (cpu_target - 1); i > 0; i--) { | |
| if (usable_cpu_array[i] == 0) | |
| continue; | |
| if (usable_cpu_array[i] > rem_nodes) | |
| usable_cpu_array[i] = rem_nodes; | |
| if (rem_nodes > 0) { | |
| rem_nodes -= usable_cpu_array[i]; | |
| rem_cpus -= (usable_cpu_array[i] * i); | |
| } | |
| } | |
| if ((rem_cpus > 0) || (rem_nodes > 0)){ /* Can not satisfy request */ | |
| xfree(usable_cpu_array); | |
| FREE_NULL_BITMAP(picked_node_bitmap); | |
| return NULL; | |
| } | |
| rem_nodes = save_rem_nodes; | |
| rem_cpus = save_rem_cpus; | |
| /* Pick nodes with CPU counts below original target */ | |
| check_cnt = 0; | |
| while (_next_node_inx(&pos, &check_cnt, order_cnt, nodes_bitmap, | |
| start_pos, order_map, order_cnt, | |
| &node_inx) == SLURM_SUCCESS) { | |
| if (usable_cpu_cnt[node_inx] >= cpu_target) | |
| continue; /* already picked */ | |
| if (usable_cpu_array[usable_cpu_cnt[node_inx]] == 0) | |
| continue; | |
| usable_cpu_array[usable_cpu_cnt[node_inx]]--; | |
| bit_set(picked_node_bitmap, node_inx); | |
| rem_cpus -= usable_cpu_cnt[node_inx]; | |
| rem_nodes--; | |
| if ((rem_cpus <= 0) && (rem_nodes <= 0)) { | |
| /* Satisfied request */ | |
| job_ptr->job_resrcs->next_step_node_inx = | |
| _next_rank_start(order_map, order_cnt, pos); | |
| xfree(usable_cpu_array); | |
| return picked_node_bitmap; | |
| } | |
| if (rem_nodes == 0) /* Reached node limit */ | |
| break; | |
| } | |
| /* Can not satisfy request */ | |
| xfree(usable_cpu_array); | |
| FREE_NULL_BITMAP(picked_node_bitmap); | |
| return NULL; | |
| } | |
| static int _mark_busy_nodes(void *x, void *arg) | |
| { | |
| step_record_t *step_ptr = (step_record_t *) x; | |
| bitstr_t *nodes_idle = arg; | |
| if (step_ptr->state < JOB_RUNNING) | |
| return 0; | |
| /* | |
| * Don't consider the batch and extern steps when | |
| * looking for "idle" nodes. | |
| */ | |
| if ((step_ptr->step_id.step_id == SLURM_BATCH_SCRIPT) || | |
| (step_ptr->step_id.step_id == SLURM_EXTERN_CONT) || | |
| (step_ptr->step_id.step_id == SLURM_INTERACTIVE_STEP) || | |
| (step_ptr->flags & SSF_EXT_LAUNCHER)) | |
| return 0; | |
| if (!step_ptr->step_node_bitmap) { | |
| error("%s: %pS has no step_node_bitmap", | |
| __func__, step_ptr); | |
| return 0; | |
| } | |
| bit_and_not(nodes_idle, step_ptr->step_node_bitmap); | |
| if (slurm_conf.debug_flags & DEBUG_FLAG_STEPS) { | |
| char *temp; | |
| temp = bitmap2node_name(step_ptr->step_node_bitmap); | |
| log_flag(STEPS, "%s: %pS has nodes %s", | |
| __func__, step_ptr, temp); | |
| xfree(temp); | |
| } | |
| return 0; | |
| } | |
| static void _step_test_gres(step_create_req_msg_t *step_spec, | |
| gres_stepmgr_step_test_args_t *gres_test_args, | |
| job_record_t *job_ptr, | |
| uint32_t *node_usable_cpu_cnt, uint32_t *total_cpus, | |
| uint32_t *avail_cpus, int *gres_invalid_nodes, | |
| int *fail_mode) | |
| { | |
| uint64_t gres_cpus; | |
| int err_code = SLURM_SUCCESS; | |
| gres_test_args->err_code = &err_code; | |
| /* ignore current step allocations */ | |
| gres_test_args->ignore_alloc = true; | |
| gres_cpus = gres_stepmgr_step_test(gres_test_args); | |
| *total_cpus = MIN(*total_cpus, gres_cpus); | |
| /* | |
| * consider current step allocations if | |
| * not --overlap=force | |
| */ | |
| if (!(step_spec->flags & SSF_OVERLAP_FORCE)) { | |
| gres_test_args->ignore_alloc = false; | |
| gres_cpus = gres_stepmgr_step_test(gres_test_args); | |
| } | |
| if (gres_cpus < *avail_cpus) { | |
| log_flag(STEPS, "%s: %pJ Usable CPUs for GRES %"PRIu64" from %d previously available", | |
| __func__, job_ptr, gres_cpus, | |
| *avail_cpus); | |
| *avail_cpus = gres_cpus; | |
| *node_usable_cpu_cnt = *avail_cpus; | |
| if (err_code != SLURM_SUCCESS) | |
| *fail_mode = err_code; | |
| else | |
| *fail_mode = ESLURM_INVALID_GRES; | |
| if (*total_cpus == 0) { | |
| /* | |
| * total_cpus == 0 is set from this: | |
| * MIN(*total_cpus, gres_cpus); | |
| * This means that it is impossible to run this step on | |
| * this node due to GRES. | |
| */ | |
| *gres_invalid_nodes = *gres_invalid_nodes + 1; | |
| } | |
| } | |
| } | |
| /* Returns threads_per_core required by the step or NO_VAL16 if not specified */ | |
| static uint16_t _get_threads_per_core(uint16_t step_threads_per_core, | |
| job_record_t *job_ptr) | |
| { | |
| uint16_t tpc = NO_VAL16; | |
| if (step_threads_per_core && | |
| (step_threads_per_core != NO_VAL16)) { | |
| tpc = step_threads_per_core; | |
| } else if (job_ptr->details->mc_ptr->threads_per_core && | |
| (job_ptr->details->mc_ptr->threads_per_core != NO_VAL16)) | |
| tpc = job_ptr->details->mc_ptr->threads_per_core; | |
| return tpc; | |
| } | |
| static int _cmp_cpu_counts(const void *num1, const void *num2) { | |
| uint16_t cpu1 = *(uint16_t *) num1; | |
| uint16_t cpu2 = *(uint16_t *) num2; | |
| if (cpu1 > cpu2) | |
| return -1; | |
| else if (cpu1 < cpu2) | |
| return 1; | |
| return 0; | |
| } | |
| static void _set_max_num_tasks(step_create_req_msg_t *step_spec, | |
| job_record_t *job_ptr, bitstr_t *node_bitmap, | |
| int cpus_per_task) | |
| { | |
| int j = 0; | |
| int k = 0; | |
| uint32_t avail_cnt, num_nodes; | |
| uint16_t *cpus; | |
| uint32_t num_tasks = 0; | |
| uint16_t tpc = _get_threads_per_core(step_spec->threads_per_core, | |
| job_ptr); | |
| xassert(node_bitmap); | |
| xassert(cpus_per_task); | |
| avail_cnt = bit_set_count(node_bitmap); | |
| num_nodes = MIN(avail_cnt, step_spec->max_nodes); | |
| cpus = xcalloc(avail_cnt, sizeof(*cpus)); | |
| for (int i = 0; i < job_ptr->job_resrcs->nhosts; i++) { | |
| j = bit_ffs_from_bit(job_ptr->job_resrcs->node_bitmap, j); | |
| if (j < 0) | |
| break; | |
| if (!bit_test(node_bitmap, j)) { | |
| j++; | |
| continue; | |
| } | |
| if (tpc != NO_VAL16) { | |
| cpus[k] = ROUNDUP(job_ptr->job_resrcs->cpus[i], | |
| node_record_table_ptr[j]->tpc); | |
| cpus[k] *= tpc; | |
| } else | |
| cpus[k] = job_ptr->job_resrcs->cpus[i]; | |
| j++; | |
| k++; | |
| } | |
| if (num_nodes < avail_cnt) | |
| qsort(cpus, avail_cnt, sizeof(*cpus), _cmp_cpu_counts); | |
| for (int i = 0; i < num_nodes; i++) { | |
| num_tasks += cpus[i] / cpus_per_task; | |
| } | |
| step_spec->num_tasks = num_tasks; | |
| step_spec->cpu_count = num_tasks * cpus_per_task; | |
| xfree(cpus); | |
| } | |
| /* | |
| * _pick_step_nodes - select nodes for a job step that satisfy its requirements | |
| * we satisfy the super-set of constraints. | |
| * IN job_ptr - pointer to job to have new step started | |
| * IN step_spec - job step specification | |
| * IN step_gres_list - job step's gres requirement details | |
| * IN cpus_per_task - NOTE could be zero | |
| * IN node_count - How many real nodes a select plugin should be looking for | |
| * OUT return_code - exit code or SLURM_SUCCESS | |
| * global: node_record_table_ptr - pointer to global node table | |
| * NOTE: returns all of a job's nodes if step_spec->node_count == INFINITE | |
| * NOTE: returned bitmap must be freed by the caller using FREE_NULL_BITMAP() | |
| */ | |
| static bitstr_t *_pick_step_nodes(job_record_t *job_ptr, | |
| step_create_req_msg_t *step_spec, | |
| list_t *step_gres_list, int cpus_per_task, | |
| uint32_t node_count, int *return_code) | |
| { | |
| node_record_t *node_ptr; | |
| bitstr_t *nodes_avail = NULL, *nodes_idle = NULL; | |
| bitstr_t *nodes_picked = NULL, *node_tmp = NULL; | |
| int error_code, nodes_picked_cnt = 0, cpus_picked_cnt = 0; | |
| int cpu_cnt; | |
| int mem_blocked_nodes = 0, mem_blocked_cpus = 0; | |
| int job_blocked_nodes = 0, job_blocked_cpus = 0; | |
| int gres_invalid_nodes = 0; | |
| job_resources_t *job_resrcs_ptr = job_ptr->job_resrcs; | |
| uint32_t *usable_cpu_cnt = NULL; | |
| uint32_t orig_next_step_node_inx = 0; | |
| gres_stepmgr_step_test_args_t gres_test_args = { | |
| .cpus_per_task = cpus_per_task, | |
| .first_step_node = true, | |
| .job_gres_list = job_ptr->gres_list_alloc, | |
| .job_id = job_ptr->job_id, | |
| .job_resrcs_ptr = job_resrcs_ptr, | |
| .max_rem_nodes = step_spec->max_nodes, | |
| .step_gres_list = step_gres_list, | |
| .step_id = NO_VAL, | |
| .test_mem = false, | |
| }; | |
| xassert(job_resrcs_ptr); | |
| xassert(job_resrcs_ptr->cpus); | |
| xassert(job_resrcs_ptr->cpus_used); | |
| orig_next_step_node_inx = job_resrcs_ptr->next_step_node_inx; | |
| *return_code = SLURM_SUCCESS; | |
| if (job_ptr->node_bitmap == NULL) { | |
| *return_code = ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE; | |
| return NULL; | |
| } | |
| if (step_spec->max_nodes == 0) | |
| step_spec->max_nodes = job_ptr->node_cnt; | |
| if (step_spec->max_nodes < step_spec->min_nodes) { | |
| *return_code = ESLURM_INVALID_NODE_COUNT; | |
| return NULL; | |
| } | |
| nodes_avail = bit_copy(job_ptr->node_bitmap); | |
| bit_and(nodes_avail, stepmgr_ops->up_node_bitmap); | |
| if (step_spec->exc_nodes) { | |
| bitstr_t *exc_bitmap = NULL; | |
| error_code = node_name2bitmap(step_spec->exc_nodes, false, | |
| &exc_bitmap, NULL); | |
| if (error_code) | |
| debug("Assuming invalid nodename requested for exclusion from step is excluded"); | |
| bit_and_not(nodes_avail, exc_bitmap); | |
| if (step_spec->node_list) { | |
| bitstr_t *req_nodes = NULL; | |
| error_code = node_name2bitmap( | |
| step_spec->node_list, false, | |
| &req_nodes, NULL); | |
| if (error_code) { | |
| info("%s: invalid requested node list %s", | |
| __func__, | |
| step_spec->node_list); | |
| FREE_NULL_BITMAP(exc_bitmap); | |
| FREE_NULL_BITMAP(req_nodes); | |
| goto cleanup; | |
| } | |
| if (bit_overlap_any(req_nodes, exc_bitmap)) { | |
| info("%s: %ps requested nodes %s is also excluded %s", | |
| __func__, &step_spec->step_id, | |
| step_spec->node_list, | |
| step_spec->exc_nodes); | |
| FREE_NULL_BITMAP(exc_bitmap); | |
| FREE_NULL_BITMAP(req_nodes); | |
| goto cleanup; | |
| } | |
| FREE_NULL_BITMAP(req_nodes); | |
| } | |
| FREE_NULL_BITMAP(exc_bitmap); | |
| } | |
| if (step_spec->features && | |
| (!job_ptr->details || | |
| xstrcmp(step_spec->features, job_ptr->details->features_use))) { | |
| /* | |
| * We only select for a single feature name here. | |
| * Ignore step features if equal to job features. | |
| * FIXME: Add support for AND, OR, etc. here if desired | |
| */ | |
| node_feature_t *feat_ptr; | |
| feat_ptr = | |
| list_find_first(active_feature_list, list_find_feature, | |
| (void *) step_spec->features); | |
| if (feat_ptr && feat_ptr->node_bitmap) | |
| bit_and(nodes_avail, feat_ptr->node_bitmap); | |
| else { | |
| bit_clear_all(nodes_avail); | |
| *return_code = ESLURM_INVALID_FEATURE; | |
| goto cleanup; | |
| } | |
| } | |
| if (step_spec->pn_min_memory && | |
| ((job_resrcs_ptr->memory_allocated == NULL) || | |
| (job_resrcs_ptr->memory_used == NULL))) { | |
| error("%s: job lacks memory allocation details to enforce memory limits for %pJ", | |
| __func__, job_ptr); | |
| step_spec->pn_min_memory = 0; | |
| } else if (step_spec->pn_min_memory == MEM_PER_CPU) | |
| step_spec->pn_min_memory = 0; /* clear MEM_PER_CPU flag */ | |
| if (job_ptr->next_step_id == 0) { | |
| for (int i = 0; | |
| (node_ptr = next_node_bitmap(job_ptr->node_bitmap, &i)); | |
| i++) { | |
| if (IS_NODE_POWERED_DOWN(node_ptr) || | |
| IS_NODE_FUTURE(node_ptr) || | |
| IS_NODE_NO_RESPOND(node_ptr)) { | |
| /* | |
| * Node is/was powered down. Need to wait | |
| * for it to start responding again. | |
| */ | |
| FREE_NULL_BITMAP(nodes_avail); | |
| *return_code = ESLURM_NODES_BUSY; | |
| return NULL; | |
| } | |
| } | |
| /* stepd_stepmgr_ops does not set job_config_fini */ | |
| if (IS_JOB_CONFIGURING(job_ptr) && | |
| stepmgr_ops->job_config_fini) { | |
| stepmgr_ops->job_config_fini(job_ptr); | |
| if (!IS_JOB_CONFIGURING(job_ptr)) | |
| info("%s: Configuration for %pJ is complete", | |
| __func__, job_ptr); | |
| } | |
| } | |
| if (_is_mem_resv() && step_spec->pn_min_memory && | |
| ((step_spec->pn_min_memory & MEM_PER_CPU) == 0) && | |
| job_ptr->details && job_ptr->details->pn_min_memory && | |
| ((job_ptr->details->pn_min_memory & MEM_PER_CPU) == 0) && | |
| (step_spec->pn_min_memory > | |
| job_ptr->details->pn_min_memory)) { | |
| FREE_NULL_BITMAP(nodes_avail); | |
| *return_code = ESLURM_INVALID_TASK_MEMORY; | |
| return NULL; | |
| } | |
| usable_cpu_cnt = xcalloc(node_record_count, sizeof(uint32_t)); | |
| for (int i = 0, node_inx = -1; | |
| (node_ptr = next_node_bitmap(job_resrcs_ptr->node_bitmap, &i)); | |
| i++) { | |
| node_inx++; | |
| if (!bit_test(nodes_avail, i)) | |
| continue; /* node now DOWN */ | |
| usable_cpu_cnt[i] = job_resrcs_ptr->cpus[node_inx]; | |
| log_flag(STEPS, "%s: %pJ Currently running steps use %d of allocated %d CPUs on node %s", | |
| __func__, job_ptr, | |
| job_resrcs_ptr->cpus_used[node_inx], | |
| usable_cpu_cnt[i], node_record_table_ptr[i]->name); | |
| /* | |
| * Don't do this test if --overlap=force or | |
| * --external-launcher | |
| */ | |
| if ((!(step_spec->flags & SSF_OVERLAP_FORCE)) && | |
| (!(step_spec->flags & SSF_EXT_LAUNCHER))) { | |
| /* | |
| * If whole is given and | |
| * job_resrcs_ptr->cpus_used[node_inx] | |
| * we can't use this node. | |
| */ | |
| if ((step_spec->flags & SSF_WHOLE) && | |
| job_resrcs_ptr->cpus_used[node_inx]) { | |
| log_flag(STEPS, "%s: %pJ Node requested --whole node while other step running here.", | |
| __func__, job_ptr); | |
| job_blocked_cpus += | |
| job_resrcs_ptr->cpus_used[node_inx]; | |
| job_blocked_nodes++; | |
| usable_cpu_cnt[i] = 0; | |
| } else { | |
| uint16_t tpc = _get_threads_per_core( | |
| step_spec->threads_per_core, job_ptr); | |
| uint16_t used = | |
| job_resrcs_ptr->cpus_used[node_inx]; | |
| uint16_t blocked; | |
| /* | |
| * --overcommit can leave cpus_used above | |
| * cpus, which would wrap this unsigned count | |
| * into a huge number of free CPUs. | |
| */ | |
| blocked = MIN(usable_cpu_cnt[i], used); | |
| usable_cpu_cnt[i] -= blocked; | |
| job_blocked_cpus += blocked; | |
| if (step_spec->threads_per_core != NO_VAL16 && | |
| (node_ptr->threads != tpc)) { | |
| log_flag(STEPS, "%s, %pJ requested threads per core does not match node defaults, adjusting usable cpu count", | |
| __func__, job_ptr); | |
| /* | |
| * calculate number of free cores and | |
| * multiply by requested threads per | |
| * core since usable_cpu_cnt will | |
| * always be a multiple of the nodes | |
| * threads_per_core this should always | |
| * be correct | |
| */ | |
| usable_cpu_cnt[i] = | |
| (usable_cpu_cnt[i] / | |
| node_ptr->threads) * tpc; | |
| } | |
| if (!usable_cpu_cnt[i]) { | |
| job_blocked_nodes++; | |
| log_flag(STEPS, "%s: %pJ Skipping node %s. Not enough CPUs to run step here.", | |
| __func__, | |
| job_ptr, | |
| node_record_table_ptr[i]->name); | |
| } | |
| } | |
| } | |
| if (!usable_cpu_cnt[i]) { | |
| bit_clear(nodes_avail, i); | |
| continue; | |
| } | |
| if ((step_spec->pn_min_memory && _is_mem_resv()) || | |
| step_gres_list) { | |
| int fail_mode = ESLURM_NODES_BUSY; | |
| uint64_t tmp_mem; | |
| uint32_t tmp_cpus, avail_cpus, total_cpus; | |
| uint32_t avail_tasks, total_tasks; | |
| gres_test_args.node_offset = node_inx; | |
| gres_test_args.test_mem = false; | |
| avail_cpus = total_cpus = usable_cpu_cnt[i];; | |
| if (_is_mem_resv() && | |
| step_spec->pn_min_memory & MEM_PER_CPU) { | |
| uint64_t mem_use = step_spec->pn_min_memory; | |
| mem_use &= (~MEM_PER_CPU); | |
| /* ignore current step allocations */ | |
| tmp_mem = job_resrcs_ptr-> | |
| memory_allocated[node_inx]; | |
| tmp_cpus = tmp_mem / mem_use; | |
| total_cpus = MIN(total_cpus, tmp_cpus); | |
| /* | |
| * consider current step allocations if | |
| * not --overlap=force | |
| */ | |
| if (!(step_spec->flags & SSF_OVERLAP_FORCE)) { | |
| tmp_mem -= job_resrcs_ptr-> | |
| memory_used[node_inx]; | |
| tmp_cpus = tmp_mem / mem_use; | |
| } | |
| if (tmp_cpus < avail_cpus) { | |
| avail_cpus = tmp_cpus; | |
| usable_cpu_cnt[i] = avail_cpus; | |
| fail_mode = ESLURM_INVALID_TASK_MEMORY; | |
| } | |
| log_flag(STEPS, "%s: %pJ Based on --mem-per-cpu=%"PRIu64" we have %d/%d usable of available cpus on node %s, usable memory was: %"PRIu64, | |
| __func__, job_ptr, mem_use, tmp_cpus, | |
| avail_cpus, node_ptr->name, tmp_mem); | |
| } else if (_is_mem_resv() && step_spec->pn_min_memory) { | |
| uint64_t mem_use = step_spec->pn_min_memory; | |
| /* ignore current step allocations */ | |
| tmp_mem = job_resrcs_ptr-> | |
| memory_allocated[node_inx]; | |
| if (tmp_mem < mem_use) | |
| total_cpus = 0; | |
| /* | |
| * consider current step allocations if | |
| * not --overlap=force | |
| */ | |
| if (!(step_spec->flags & SSF_OVERLAP_FORCE)) { | |
| tmp_mem -= job_resrcs_ptr-> | |
| memory_used[node_inx]; | |
| } | |
| if ((tmp_mem < mem_use) && (avail_cpus > 0)) { | |
| log_flag(STEPS, "%s: %pJ Usable memory on node %s: %"PRIu64" is less than requested %"PRIu64" skipping the node", | |
| __func__, job_ptr, | |
| node_ptr->name, | |
| tmp_mem, | |
| mem_use); | |
| avail_cpus = 0; | |
| usable_cpu_cnt[i] = avail_cpus; | |
| fail_mode = ESLURM_INVALID_TASK_MEMORY; | |
| } | |
| } else if (_is_mem_resv()) | |
| gres_test_args.test_mem = true; | |
| _step_test_gres(step_spec, &gres_test_args, job_ptr, | |
| &usable_cpu_cnt[i], | |
| &total_cpus, &avail_cpus, | |
| &gres_invalid_nodes, | |
| &fail_mode); | |
| avail_tasks = avail_cpus; | |
| total_tasks = total_cpus; | |
| if (cpus_per_task > 0) { | |
| avail_tasks /= cpus_per_task; | |
| total_tasks /= cpus_per_task; | |
| } | |
| if (avail_tasks == 0) { | |
| log_flag(STEPS, "%s: %pJ No task can start on node %s", | |
| __func__, job_ptr, node_ptr->name); | |
| if ((step_spec->min_nodes == INFINITE) || | |
| (step_spec->min_nodes == | |
| job_ptr->node_cnt)) { | |
| log_flag(STEPS, "%s: %pJ All nodes in allocation required, but can't use them now", | |
| __func__, job_ptr); | |
| FREE_NULL_BITMAP(nodes_avail); | |
| xfree(usable_cpu_cnt); | |
| *return_code = ESLURM_NODES_BUSY; | |
| if (total_tasks == 0 && | |
| (fail_mode != | |
| ESLURM_INVALID_TASK_MEMORY)) { | |
| *return_code = fail_mode; | |
| log_flag(STEPS, "%s: %pJ Step cannot ever run in the allocation: %s", | |
| __func__, | |
| job_ptr, | |
| slurm_strerror( | |
| fail_mode)); | |
| } | |
| return NULL; | |
| } | |
| bit_clear(nodes_avail, i); | |
| mem_blocked_nodes++; | |
| mem_blocked_cpus += (total_cpus - avail_cpus); | |
| } else { | |
| mem_blocked_cpus += (total_cpus - avail_cpus); | |
| gres_test_args.first_step_node = false; | |
| } | |
| } | |
| } | |
| if (gres_invalid_nodes > | |
| (job_resrcs_ptr->nhosts - step_spec->min_nodes)) { | |
| *return_code = ESLURM_INVALID_GRES; | |
| log_flag(STEPS, "%s: Never able to satisfy the GRES request for this step", | |
| __func__); | |
| FREE_NULL_BITMAP(nodes_avail); | |
| xfree(usable_cpu_cnt); | |
| return NULL; | |
| } | |
| if ((step_spec->min_nodes == INFINITE) && | |
| (!step_spec->node_list || | |
| ((step_spec->task_dist & SLURM_DIST_STATE_BASE) != | |
| SLURM_DIST_ARBITRARY))) { /* use all nodes */ | |
| if ((step_spec->num_tasks == NO_VAL) && | |
| !(step_spec->flags & SSF_EXT_LAUNCHER)) { | |
| _set_max_num_tasks(step_spec, job_ptr, nodes_avail, | |
| cpus_per_task); | |
| if (step_spec->num_tasks == 0) { | |
| log_flag(STEPS, "%s: Step requested more processors per task (%d) than can be satisfied.", | |
| __func__, cpus_per_task); | |
| *return_code = ESLURM_TOO_MANY_REQUESTED_CPUS; | |
| goto cleanup; | |
| } | |
| } | |
| job_resrcs_ptr->next_step_node_inx = 0; | |
| xfree(usable_cpu_cnt); | |
| return nodes_avail; | |
| } | |
| /* | |
| * An allocating srun will send in the same node_list that was already | |
| * used to construct the job allocation. In that case, we can assume | |
| * that the job allocation already satisfies those requirements. An | |
| * arbitrary list is always checked: it drives the step's task layout | |
| * and indexes the job's resources. | |
| */ | |
| if (step_spec->node_list && | |
| (xstrcmp(step_spec->node_list, job_ptr->details->req_nodes) || | |
| ((step_spec->task_dist & SLURM_DIST_STATE_BASE) == | |
| SLURM_DIST_ARBITRARY))) { | |
| bitstr_t *selected_nodes = NULL; | |
| int selected_nodes_cnt = 0; | |
| /* | |
| * An arbitrary list must be a literal list of node names, not | |
| * a hostlist function (feature{...}, switch{...}, ...). | |
| */ | |
| if (((step_spec->task_dist & SLURM_DIST_STATE_BASE) == | |
| SLURM_DIST_ARBITRARY) && | |
| xstrchr(step_spec->node_list, '{')) { | |
| log_flag(STEPS, "%s: invalid node list %s for arbitrary distribution", | |
| __func__, step_spec->node_list); | |
| *return_code = ESLURM_INVALID_NODE_NAME; | |
| goto cleanup; | |
| } | |
| log_flag(STEPS, "%s: selected nodelist is %s", | |
| __func__, step_spec->node_list); | |
| error_code = node_name2bitmap(step_spec->node_list, false, | |
| &selected_nodes, NULL); | |
| if (error_code || !selected_nodes) { | |
| log_flag(STEPS, "%s: invalid node list %s", __func__, | |
| step_spec->node_list); | |
| FREE_NULL_BITMAP(selected_nodes); | |
| goto cleanup; | |
| } | |
| if (!bit_super_set(selected_nodes, job_ptr->node_bitmap)) { | |
| log_flag(STEPS, "%s: requested nodes %s not part of %pJ", | |
| __func__, step_spec->node_list, job_ptr); | |
| FREE_NULL_BITMAP(selected_nodes); | |
| goto cleanup; | |
| } | |
| if (!bit_super_set(selected_nodes, nodes_avail)) { | |
| /* | |
| * If some nodes still have some memory or CPUs | |
| * allocated to other steps, just defer the execution | |
| * of the step | |
| */ | |
| if (job_blocked_nodes) { | |
| *return_code = ESLURM_NODES_BUSY; | |
| log_flag(STEPS, "%s: some requested nodes %s still have CPUs used by other steps", | |
| __func__, step_spec->node_list); | |
| } else if (mem_blocked_nodes == 0) { | |
| *return_code = ESLURM_INVALID_TASK_MEMORY; | |
| log_flag(STEPS, "%s: requested nodes %s have inadequate memory", | |
| __func__, step_spec->node_list); | |
| } else { | |
| *return_code = ESLURM_NODES_BUSY; | |
| log_flag(STEPS, "%s: some requested nodes %s still have memory used by other steps", | |
| __func__, step_spec->node_list); | |
| } | |
| FREE_NULL_BITMAP(selected_nodes); | |
| goto cleanup; | |
| } | |
| selected_nodes_cnt = bit_set_count(selected_nodes); | |
| /* | |
| * Use selected nodes to run the step and | |
| * mark them unavailable for future use | |
| */ | |
| /* | |
| * If we have selected more than we requested | |
| * make the available nodes equal to the | |
| * selected nodes and we will pick from that | |
| * list later on in the function. | |
| * Other than that copy the nodes selected as | |
| * the nodes we want. | |
| */ | |
| if (selected_nodes_cnt > step_spec->max_nodes) { | |
| log_flag(STEPS, "%s: requested nodes %s exceed max node count for %pJ (%d > %u)", | |
| __func__, step_spec->node_list, | |
| job_ptr, selected_nodes_cnt, | |
| step_spec->max_nodes); | |
| FREE_NULL_BITMAP(selected_nodes); | |
| goto cleanup; | |
| } | |
| /* | |
| * Pin the node count to the list only after the check above, | |
| * so that a max_nodes smaller than the arbitrary list is still | |
| * rejected rather than silently overridden. | |
| */ | |
| if ((step_spec->task_dist & SLURM_DIST_STATE_BASE) == | |
| SLURM_DIST_ARBITRARY) { | |
| step_spec->min_nodes = selected_nodes_cnt; | |
| step_spec->max_nodes = selected_nodes_cnt; | |
| } | |
| if (step_spec->min_nodes && | |
| (selected_nodes_cnt > step_spec->min_nodes)) { | |
| nodes_picked = bit_alloc(bit_size(nodes_avail)); | |
| FREE_NULL_BITMAP(nodes_avail); | |
| nodes_avail = selected_nodes; | |
| selected_nodes = NULL; | |
| } else { | |
| nodes_picked = bit_copy(selected_nodes); | |
| bit_and_not(nodes_avail, selected_nodes); | |
| FREE_NULL_BITMAP(selected_nodes); | |
| } | |
| } else { | |
| nodes_picked = bit_alloc(bit_size(nodes_avail)); | |
| } | |
| /* If gres_per_step then filter nodes_avail to nodes that fill req */ | |
| gres_stepmgr_step_test_per_step(step_gres_list, job_ptr, | |
| nodes_avail, step_spec->min_nodes); | |
| /* | |
| * In case we are in relative mode, do not look for idle nodes | |
| * as we will not try to get idle nodes first but try to get | |
| * the relative node first | |
| */ | |
| if (step_spec->relative != NO_VAL16) { | |
| /* | |
| * Remove first (step_spec->relative) nodes from | |
| * available list. Relative placement must always start at the | |
| * first remaining node after the offset is applied rather than | |
| * using the cyclic next-step index from prior allocations. | |
| */ | |
| bitstr_t *relative_nodes = NULL; | |
| relative_nodes = bit_pick_cnt(job_ptr->node_bitmap, | |
| step_spec->relative); | |
| if (relative_nodes == NULL) { | |
| log_flag(STEPS, "%s: Invalid relative value (%u) for %pJ", | |
| __func__, step_spec->relative, job_ptr); | |
| goto cleanup; | |
| } | |
| bit_and_not(nodes_avail, relative_nodes); | |
| FREE_NULL_BITMAP(relative_nodes); | |
| job_resrcs_ptr->next_step_node_inx = 0; | |
| } else { | |
| nodes_idle = bit_copy(nodes_avail); | |
| list_for_each(job_ptr->step_list, _mark_busy_nodes, nodes_idle); | |
| } | |
| if (slurm_conf.debug_flags & DEBUG_FLAG_STEPS) { | |
| char *temp1, *temp2, *temp3; | |
| temp1 = bitmap2node_name(nodes_avail); | |
| temp2 = bitmap2node_name(nodes_idle); | |
| if (step_spec->node_list) | |
| temp3 = step_spec->node_list; | |
| else | |
| temp3 = "NONE"; | |
| log_flag(STEPS, "%s: step pick %u-%u nodes, avail:%s idle:%s picked:%s", | |
| __func__, step_spec->min_nodes, step_spec->max_nodes, | |
| temp1, temp2, temp3); | |
| xfree(temp1); | |
| xfree(temp2); | |
| } | |
| if ((step_spec->num_tasks == NO_VAL) && | |
| !(step_spec->flags & SSF_EXT_LAUNCHER)) { | |
| uint32_t cnt = 0; | |
| bitstr_t *node_bitmap = NULL; | |
| if ((step_spec->flags & SSF_OVERLAP_FORCE) || | |
| (step_spec->relative != NO_VAL16)) { | |
| cnt = bit_set_count(nodes_avail); | |
| node_bitmap = nodes_avail; | |
| } else if (nodes_idle) { | |
| cnt = bit_set_count(nodes_idle); | |
| node_bitmap = nodes_idle; | |
| } | |
| if (cnt < step_spec->min_nodes) { | |
| log_flag(STEPS, "%s: Step requested more nodes (%u) than are available (%d), deferring step until enough nodes are available.", | |
| __func__, step_spec->min_nodes, cnt); | |
| *return_code = ESLURM_NODES_BUSY; | |
| goto cleanup; | |
| } | |
| _set_max_num_tasks(step_spec, job_ptr, node_bitmap, | |
| cpus_per_task); | |
| if (step_spec->num_tasks == 0) { | |
| log_flag(STEPS, "%s: Step requested more processors per task (%d) than can be satisfied.", | |
| __func__, cpus_per_task); | |
| *return_code = ESLURM_TOO_MANY_REQUESTED_CPUS; | |
| goto cleanup; | |
| } | |
| } | |
| /* | |
| * If user specifies step needs a specific processor count and | |
| * all nodes have the same processor count, just translate this to | |
| * a node count | |
| */ | |
| if (step_spec->cpu_count && job_resrcs_ptr && | |
| (job_resrcs_ptr->cpu_array_cnt == 1) && | |
| (job_resrcs_ptr->cpu_array_value)) { | |
| uint32_t cpu_count = step_spec->cpu_count; | |
| uint16_t req_tpc; | |
| uint32_t nodes_needed; | |
| /* | |
| * Expand cpu account to account for blocked/used threads when | |
| * using threads-per-core. See _step_[de]alloc_lps() for similar | |
| * code. | |
| */ | |
| req_tpc = _get_threads_per_core(step_spec->threads_per_core, | |
| job_ptr); | |
| /* | |
| * Only process this differently if the allocation requested | |
| * more threads per core than the step is requesting as | |
| * job_resrcs->cpu_array_value is already processed with the | |
| * threads per core the allocation requested so you don't need | |
| * to do this again. See src/common/job_resources.c | |
| * build_job_resources_cpu_array(). | |
| */ | |
| if ((req_tpc != NO_VAL16) && | |
| (req_tpc < job_resrcs_ptr->threads_per_core)) { | |
| int first_inx = bit_ffs(job_resrcs_ptr->node_bitmap); | |
| if (first_inx == -1) { | |
| error("%s: Job %pJ doesn't have any nodes in it! This should never happen", | |
| __func__, job_ptr); | |
| *return_code = ESLURM_INVALID_NODE_COUNT; | |
| goto cleanup; | |
| } | |
| if (req_tpc < node_record_table_ptr[first_inx]->tpc) { | |
| cpu_count = ROUNDUP(cpu_count, req_tpc); | |
| cpu_count *= | |
| node_record_table_ptr[first_inx]->tpc; | |
| } else if (req_tpc > | |
| node_record_table_ptr[first_inx]->tpc) { | |
| log_flag(STEPS, "%s: requested more threads per core than possible in allocation (%u > %u) for %pJ", | |
| __func__, | |
| req_tpc, | |
| node_record_table_ptr[first_inx]->tpc, | |
| job_ptr); | |
| *return_code = ESLURM_BAD_THREAD_PER_CORE; | |
| goto cleanup; | |
| } | |
| } | |
| nodes_needed = | |
| ROUNDUP(cpu_count, job_resrcs_ptr->cpu_array_value[0]); | |
| step_spec->min_nodes = MAX(nodes_needed, step_spec->min_nodes); | |
| /* | |
| * If we are trying to pack the nodes we only want the minimum | |
| * it takes to satisfy the request. | |
| */ | |
| if (step_spec->task_dist & SLURM_DIST_PACK_NODES) | |
| step_spec->max_nodes = step_spec->min_nodes; | |
| if (step_spec->max_nodes < step_spec->min_nodes) { | |
| log_flag(STEPS, "%s: %pJ max node less than min node count (%u < %u)", | |
| __func__, job_ptr, step_spec->max_nodes, | |
| step_spec->min_nodes); | |
| *return_code = ESLURM_TOO_MANY_REQUESTED_CPUS; | |
| goto cleanup; | |
| } | |
| } | |
| if (step_spec->min_nodes) { | |
| int cpus_needed, node_avail_cnt, nodes_needed; | |
| nodes_picked_cnt = bit_set_count(nodes_picked); | |
| log_flag(STEPS, "%s: step picked %d of %u nodes", | |
| __func__, nodes_picked_cnt, step_spec->min_nodes); | |
| /* | |
| * First do a basic test - if there aren't enough nodes for | |
| * this step to run on then we need to defer execution of this | |
| * step. As long as there aren't enough nodes for this | |
| * step we can never test if the step requested too | |
| * many CPUs, too much memory, etc. so we just bail right here. | |
| */ | |
| node_avail_cnt = bit_set_count(nodes_avail); | |
| if ((node_avail_cnt + nodes_picked_cnt) < | |
| step_spec->min_nodes) { | |
| log_flag(STEPS, "%s: Step requested more nodes (%u) than are available (%d), deferring step until enough nodes are available.", | |
| __func__, step_spec->min_nodes, | |
| node_avail_cnt); | |
| *return_code = ESLURM_NODES_BUSY; | |
| goto cleanup; | |
| } | |
| if (nodes_idle) | |
| node_avail_cnt = bit_set_count(nodes_idle); | |
| else | |
| node_avail_cnt = 0; | |
| nodes_needed = step_spec->min_nodes - nodes_picked_cnt; | |
| if ((nodes_needed > 0) && | |
| (node_avail_cnt >= nodes_needed)) { | |
| cpus_needed = _opt_cpu_cnt(step_spec->cpu_count, | |
| nodes_picked, | |
| usable_cpu_cnt); | |
| nodes_needed = _opt_node_cnt(step_spec->min_nodes, | |
| step_spec->max_nodes, | |
| node_avail_cnt, | |
| nodes_picked_cnt); | |
| node_tmp = | |
| _pick_step_nodes_cpus(job_ptr, nodes_idle, | |
| nodes_needed, cpus_needed, | |
| usable_cpu_cnt); | |
| if (node_tmp) { | |
| bit_or(nodes_picked, node_tmp); | |
| bit_and_not(nodes_idle, node_tmp); | |
| bit_and_not(nodes_avail, node_tmp); | |
| FREE_NULL_BITMAP(node_tmp); | |
| nodes_picked_cnt = step_spec->min_nodes; | |
| nodes_needed = 0; | |
| } | |
| } | |
| node_avail_cnt = bit_set_count(nodes_avail); | |
| if ((nodes_needed > 0) && | |
| (node_avail_cnt >= nodes_needed)) { | |
| cpus_needed = _opt_cpu_cnt(step_spec->cpu_count, | |
| nodes_picked, | |
| usable_cpu_cnt); | |
| nodes_needed = _opt_node_cnt(step_spec->min_nodes, | |
| step_spec->max_nodes, | |
| node_avail_cnt, | |
| nodes_picked_cnt); | |
| node_tmp = | |
| _pick_step_nodes_cpus(job_ptr, nodes_avail, | |
| nodes_needed, cpus_needed, | |
| usable_cpu_cnt); | |
| if (node_tmp == NULL) { | |
| /* Count of nodes already picked for step */ | |
| int pick_node_cnt = bit_set_count(nodes_avail); | |
| pick_node_cnt += nodes_picked_cnt; | |
| if ((step_spec->max_nodes <= pick_node_cnt) && | |
| (mem_blocked_cpus == 0) && | |
| (job_blocked_cpus == 0)) { | |
| *return_code = | |
| ESLURM_TOO_MANY_REQUESTED_CPUS; | |
| } else if ((mem_blocked_cpus > 0) || | |
| (step_spec->min_nodes <= | |
| (pick_node_cnt + mem_blocked_nodes + | |
| job_blocked_nodes))) { | |
| *return_code = ESLURM_NODES_BUSY; | |
| } else if (!bit_super_set(job_ptr->node_bitmap, | |
| stepmgr_ops->up_node_bitmap)) { | |
| *return_code = ESLURM_NODE_NOT_AVAIL; | |
| } | |
| goto cleanup; | |
| } | |
| bit_or(nodes_picked, node_tmp); | |
| bit_and_not(nodes_avail, node_tmp); | |
| FREE_NULL_BITMAP(node_tmp); | |
| nodes_picked_cnt = step_spec->min_nodes; | |
| } else if (nodes_needed > 0) { | |
| if ((step_spec->max_nodes <= nodes_picked_cnt) && | |
| (mem_blocked_cpus == 0) && | |
| (job_blocked_cpus == 0)) { | |
| *return_code = ESLURM_TOO_MANY_REQUESTED_CPUS; | |
| } else if ((mem_blocked_cpus > 0) || | |
| (step_spec->min_nodes <= | |
| (nodes_picked_cnt + mem_blocked_nodes + | |
| job_blocked_nodes))) { | |
| *return_code = ESLURM_NODES_BUSY; | |
| } else if (!bit_super_set(job_ptr->node_bitmap, | |
| stepmgr_ops->up_node_bitmap)) { | |
| *return_code = ESLURM_NODE_NOT_AVAIL; | |
| } | |
| goto cleanup; | |
| } | |
| } | |
| if (step_spec->cpu_count) { | |
| /* make sure the selected nodes have enough cpus */ | |
| cpus_picked_cnt = _count_cpus(job_ptr, nodes_picked, | |
| usable_cpu_cnt); | |
| if ((step_spec->cpu_count > cpus_picked_cnt) && | |
| (step_spec->max_nodes > nodes_picked_cnt)) { | |
| int node_idx = 0; | |
| /* Attempt to add more nodes to allocation */ | |
| nodes_picked_cnt = bit_set_count(nodes_picked); | |
| while (step_spec->cpu_count > cpus_picked_cnt) { | |
| node_idx = | |
| bit_ffs_from_bit(nodes_avail, node_idx); | |
| if (node_idx < 0) | |
| break; | |
| cpu_cnt = usable_cpu_cnt[node_idx]; | |
| bit_clear(nodes_avail, node_idx); | |
| if (cpu_cnt == 0) { | |
| /* | |
| * Node not usable (memory insufficient | |
| * to allocate any CPUs, etc.) | |
| */ | |
| continue; | |
| } | |
| bit_set(nodes_picked, node_idx); | |
| nodes_picked_cnt += 1; | |
| if (step_spec->min_nodes) | |
| step_spec->min_nodes = nodes_picked_cnt; | |
| cpus_picked_cnt += cpu_cnt; | |
| if (nodes_picked_cnt >= step_spec->max_nodes) | |
| break; | |
| } | |
| } | |
| /* | |
| * User is requesting more cpus than we got from the | |
| * picked nodes. We should return with an error | |
| */ | |
| if (step_spec->cpu_count > cpus_picked_cnt) { | |
| if (step_spec->cpu_count && | |
| (step_spec->cpu_count <= | |
| (cpus_picked_cnt + mem_blocked_cpus + | |
| job_blocked_cpus))) { | |
| *return_code = ESLURM_NODES_BUSY; | |
| } else if (!bit_super_set(job_ptr->node_bitmap, | |
| stepmgr_ops->up_node_bitmap)) { | |
| *return_code = ESLURM_NODE_NOT_AVAIL; | |
| } | |
| log_flag(STEPS, "Have %d nodes with %d cpus which is less than what the user is asking for (%d cpus) aborting.", | |
| nodes_picked_cnt, | |
| cpus_picked_cnt, | |
| step_spec->cpu_count); | |
| goto cleanup; | |
| } | |
| } | |
| FREE_NULL_BITMAP(nodes_avail); | |
| FREE_NULL_BITMAP(nodes_idle); | |
| xfree(usable_cpu_cnt); | |
| return nodes_picked; | |
| cleanup: | |
| FREE_NULL_BITMAP(nodes_avail); | |
| FREE_NULL_BITMAP(nodes_idle); | |
| FREE_NULL_BITMAP(nodes_picked); | |
| xfree(usable_cpu_cnt); | |
| job_resrcs_ptr->next_step_node_inx = orig_next_step_node_inx; | |
| if (*return_code == SLURM_SUCCESS) { | |
| *return_code = ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE; | |
| } else if (*return_code == ESLURM_NODE_NOT_AVAIL) { | |
| /* | |
| * Return ESLURM_NODES_BUSY if the node is not responding. | |
| * The node will eventually either come back UP or go DOWN. | |
| */ | |
| nodes_picked = bit_copy(stepmgr_ops->up_node_bitmap); | |
| bit_not(nodes_picked); | |
| bit_and(nodes_picked, job_ptr->node_bitmap); | |
| for (int i = 0; | |
| (node_ptr = next_node_bitmap(job_resrcs_ptr->node_bitmap, | |
| &i)); | |
| i++) { | |
| if (!IS_NODE_NO_RESPOND(node_ptr)) { | |
| *return_code = ESLURM_NODES_BUSY; | |
| break; | |
| } | |
| } | |
| FREE_NULL_BITMAP(nodes_picked); | |
| } | |
| return NULL; | |
| } | |
| /* | |
| * _count_cpus - report how many cpus are allocated to this job for the | |
| * identified nodes | |
| * IN job_ptr - point to job | |
| * IN bitmap - map of nodes to tally | |
| * IN usable_cpu_cnt - count of usable CPUs based upon memory or gres specs | |
| * RET cpu count | |
| */ | |
| static int _count_cpus(job_record_t *job_ptr, bitstr_t *bitmap, | |
| uint32_t *usable_cpu_cnt) | |
| { | |
| int i, sum = 0; | |
| node_record_t *node_ptr; | |
| xassert(usable_cpu_cnt); | |
| xassert(job_ptr->job_resrcs); | |
| xassert(job_ptr->job_resrcs->node_bitmap); | |
| for (i = 0; | |
| (node_ptr = | |
| next_node_bitmap(job_ptr->job_resrcs->node_bitmap, &i)); | |
| i++) { | |
| if (!bit_test(job_ptr->node_bitmap, node_ptr->index) || | |
| !bit_test(bitmap, node_ptr->index)) { | |
| /* absent from current job or step bitmap */ | |
| continue; | |
| } | |
| sum += usable_cpu_cnt[node_ptr->index]; | |
| } | |
| return sum; | |
| } | |
| /* Clear avail_core_bitmap cores which are not bound to the allocated gres */ | |
| static int _gres_filter_avail_cores(void *x, void *arg) | |
| { | |
| gres_state_t *gres_state_step = x; | |
| foreach_gres_filter_t *args = arg; | |
| gres_step_state_t *gres_ss = gres_state_step->gres_data; | |
| bitstr_t *filter_core_bitmap; | |
| gres_state_t *gres_state_node; | |
| gres_node_state_t *gres_ns; | |
| /* Bail early if this GRES isn't used on the node */ | |
| if (!gres_ss->gres_cnt_node_alloc || | |
| !gres_ss->gres_cnt_node_alloc[args->job_node_inx]) | |
| return 0; | |
| if (!(gres_state_node = list_find_first(args->node_gres_list, | |
| gres_find_id, | |
| &gres_state_step->plugin_id))) { | |
| error("No node gres when step gres is allocated. This should never happen."); | |
| return 0; | |
| } | |
| gres_ns = gres_state_node->gres_data; | |
| if (!gres_ns->topo_cnt) /* No topology info */ | |
| return 0; | |
| filter_core_bitmap = bit_copy(args->all_gres_core_bitmap); | |
| /* Determine which specific cores can be used */ | |
| for (int i = 0; i < gres_ns->topo_cnt; i++) { | |
| /* Is this gres allocated to the step? */ | |
| if (gres_ss->gres_bit_alloc && | |
| !bit_overlap_any( | |
| gres_ss->gres_bit_alloc[args->job_node_inx], | |
| gres_ns->topo_gres_bitmap[i])) | |
| continue; | |
| /* Does it specify which cores which can use it */ | |
| if (!gres_ns->topo_core_bitmap[i]) { | |
| bit_nset(args->any_gres_core_bitmap, | |
| args->core_start_bit, args->core_end_bit); | |
| continue; | |
| } | |
| bit_nclear(filter_core_bitmap, args->core_start_bit, | |
| args->core_end_bit); | |
| for (int j = 0; | |
| j < bit_size(gres_ns->topo_core_bitmap[i]); | |
| j++) { | |
| if (bit_test(gres_ns->topo_core_bitmap[i], j)) { | |
| bit_set(filter_core_bitmap, | |
| args->core_start_bit + j); | |
| } | |
| } | |
| bit_or(args->any_gres_core_bitmap, filter_core_bitmap); | |
| bit_and(args->all_gres_core_bitmap, filter_core_bitmap); | |
| } | |
| FREE_NULL_BITMAP(filter_core_bitmap); | |
| return 0; | |
| } | |
| /* Return true if a core was picked, false if not */ | |
| static bool _pick_step_core(step_record_t *step_ptr, | |
| job_resources_t *job_resrcs_ptr, | |
| bitstr_t *avail_core_bitmap, int job_node_inx, | |
| int sock_inx, int core_inx, bool use_all_cores) | |
| { | |
| int bit_offset; | |
| bit_offset = get_job_resources_offset(job_resrcs_ptr, | |
| job_node_inx, | |
| sock_inx, | |
| core_inx); | |
| if (bit_offset < 0) | |
| fatal("get_job_resources_offset"); | |
| if (!bit_test(avail_core_bitmap, bit_offset)) | |
| return false; | |
| if (bit_test(step_ptr->core_bitmap_job, bit_offset)) | |
| return false; /* already taken by this step */ | |
| /* Check and set the job's used cores. */ | |
| if (!(step_ptr->flags & SSF_OVERLAP_FORCE)) { | |
| if ((use_all_cores == false) && | |
| bit_test(job_resrcs_ptr->core_bitmap_used, bit_offset)) | |
| return false; | |
| bit_set(job_resrcs_ptr->core_bitmap_used, bit_offset); | |
| } | |
| log_flag(STEPS, "%s: alloc Node:%d Socket:%d Core:%d", | |
| __func__, job_node_inx, sock_inx, core_inx); | |
| bit_set(step_ptr->core_bitmap_job, bit_offset); | |
| return true; | |
| } | |
| static bool _handle_core_select(step_record_t *step_ptr, | |
| job_resources_t *job_resrcs_ptr, | |
| bitstr_t *avail_core_bitmap, | |
| int job_node_inx, uint16_t sockets, | |
| uint16_t cores, bool use_all_cores, | |
| int *core_cnt, uint16_t cores_per_task) | |
| { | |
| int core_inx, sock_inx; | |
| xassert(core_cnt); | |
| if (*core_cnt <= 0) | |
| return true; | |
| /* | |
| * Figure out the task distribution. The default is to cyclically | |
| * distribute to sockets. | |
| */ | |
| if (step_ptr->step_layout && | |
| ((step_ptr->step_layout->task_dist & SLURM_DIST_SOCKMASK) == | |
| SLURM_DIST_SOCKBLOCK)) { | |
| /* Fill sockets before allocating to the next socket */ | |
| for (sock_inx=0; sock_inx < sockets; sock_inx++) { | |
| for (core_inx = 0; core_inx < cores; core_inx++) { | |
| if (!_pick_step_core(step_ptr, job_resrcs_ptr, | |
| avail_core_bitmap, | |
| job_node_inx, sock_inx, | |
| core_inx, use_all_cores)) | |
| continue; | |
| if (--(*core_cnt) == 0) | |
| return true; | |
| } | |
| } | |
| } else if (step_ptr->step_layout && | |
| ((step_ptr->step_layout->task_dist & SLURM_DIST_SOCKMASK) == | |
| SLURM_DIST_SOCKCFULL)) { | |
| for (core_inx = 0; core_inx < cores; core_inx++) { | |
| for (sock_inx = 0; sock_inx < sockets; sock_inx++) { | |
| if (!_pick_step_core(step_ptr, job_resrcs_ptr, | |
| avail_core_bitmap, | |
| job_node_inx, sock_inx, | |
| core_inx, use_all_cores)) { | |
| if (sock_inx == sockets) | |
| sock_inx = 0; | |
| continue; | |
| } | |
| if (--(*core_cnt) == 0) | |
| return true; | |
| } | |
| } | |
| } else { /* SLURM_DIST_SOCKCYCLIC */ | |
| int task_alloc_cores = 0; | |
| int *next_core = xcalloc(sockets, sizeof(int)); | |
| bool nothing_allocated = false; | |
| while (!nothing_allocated) { | |
| nothing_allocated = true; | |
| for (sock_inx = 0; sock_inx < sockets; sock_inx++) { | |
| for (core_inx = next_core[sock_inx]; | |
| core_inx < cores; core_inx++) { | |
| next_core[sock_inx] = core_inx + 1; | |
| if (!_pick_step_core( | |
| step_ptr, | |
| job_resrcs_ptr, | |
| avail_core_bitmap, | |
| job_node_inx, | |
| sock_inx, | |
| core_inx, | |
| use_all_cores)) | |
| continue; | |
| nothing_allocated = false; | |
| if (--(*core_cnt) == 0) { | |
| xfree(next_core); | |
| return true; | |
| } | |
| if (++task_alloc_cores == | |
| cores_per_task) { | |
| task_alloc_cores = 0; | |
| break; | |
| } | |
| } | |
| } | |
| } | |
| xfree(next_core); | |
| } | |
| return false; | |
| } | |
| /* Update the step's core bitmaps, create as needed. | |
| * Add the specified task count for a specific node in the job's | |
| * and step's allocation */ | |
| static int _pick_step_cores(step_record_t *step_ptr, | |
| job_resources_t *job_resrcs_ptr, int job_node_inx, | |
| uint16_t task_cnt, uint16_t cpus_per_core, | |
| int node_inx, int ntasks_per_core, | |
| int gres_cpus_alloc) | |
| { | |
| uint16_t sockets, cores, cores_per_task, tasks_per_node; | |
| int core_cnt = (int) task_cnt; | |
| bool use_all_cores; | |
| bitstr_t *all_gres_core_bitmap = NULL, *any_gres_core_bitmap = NULL; | |
| int rc = SLURM_SUCCESS; | |
| int orig_core_cnt; | |
| xassert(task_cnt); | |
| if (!step_ptr->core_bitmap_job) | |
| step_ptr->core_bitmap_job = | |
| bit_alloc(bit_size(job_resrcs_ptr->core_bitmap)); | |
| if (get_job_resources_cnt(job_resrcs_ptr, job_node_inx, | |
| &sockets, &cores)) | |
| fatal("get_job_resources_cnt"); | |
| if (ntasks_per_core != INFINITE16) | |
| tasks_per_node = cores * ntasks_per_core * sockets; | |
| else | |
| tasks_per_node = cores * cpus_per_core * sockets; | |
| if (((step_ptr->flags & SSF_WHOLE) || task_cnt == (cores * sockets)) && | |
| (task_cnt <= tasks_per_node)) { | |
| use_all_cores = true; | |
| core_cnt = ROUNDUP(job_resrcs_ptr->cpus[job_node_inx], | |
| cpus_per_core); | |
| } else { | |
| use_all_cores = false; | |
| if (gres_cpus_alloc) { | |
| core_cnt = ROUNDUP(gres_cpus_alloc, cpus_per_core); | |
| } else if (step_ptr->cpus_per_task > 0) { | |
| core_cnt *= step_ptr->cpus_per_task; | |
| core_cnt = ROUNDUP(core_cnt, cpus_per_core); | |
| } | |
| log_flag(STEPS, "%s: step %pS requires %u cores on node %d with cpus_per_core=%u, available cpus from job: %u", | |
| __func__, step_ptr, core_cnt, job_node_inx, | |
| cpus_per_core, job_resrcs_ptr->cpus[job_node_inx]); | |
| if (core_cnt > ROUNDUP(job_resrcs_ptr->cpus[job_node_inx], | |
| cpus_per_core) && | |
| !(step_ptr->flags & SSF_OVERCOMMIT)) { | |
| /* Node can never fulfill step request */ | |
| return ESLURM_TOO_MANY_REQUESTED_CPUS; | |
| } | |
| } | |
| all_gres_core_bitmap = bit_copy(job_resrcs_ptr->core_bitmap); | |
| any_gres_core_bitmap = bit_copy(job_resrcs_ptr->core_bitmap); | |
| if (step_ptr->gres_list_alloc) { | |
| foreach_gres_filter_t args = { | |
| .all_gres_core_bitmap = all_gres_core_bitmap, | |
| .any_gres_core_bitmap = any_gres_core_bitmap, | |
| .core_start_bit = get_job_resources_offset( | |
| job_resrcs_ptr, job_node_inx, 0, 0), | |
| .core_end_bit = get_job_resources_offset( | |
| job_resrcs_ptr, job_node_inx, sockets - 1, | |
| cores - 1), | |
| .job_node_inx = job_node_inx, | |
| .node_gres_list = | |
| node_record_table_ptr[node_inx]->gres_list, | |
| }; | |
| if ((args.core_start_bit > bit_size(all_gres_core_bitmap)) || | |
| (args.core_end_bit > bit_size(all_gres_core_bitmap))) | |
| error("coremap offsets fall outside core_bitmap size. This should never happen."); | |
| else if (!args.node_gres_list) | |
| error("No node gres when step gres is allocated. This should never happen."); | |
| else { | |
| bit_nclear(any_gres_core_bitmap, args.core_start_bit, | |
| args.core_end_bit); | |
| list_for_each(step_ptr->gres_list_alloc, | |
| _gres_filter_avail_cores, &args); | |
| bit_and(any_gres_core_bitmap, | |
| job_resrcs_ptr->core_bitmap); | |
| } | |
| } | |
| cores_per_task = ROUNDUP(core_cnt, task_cnt); /* Round up */ | |
| orig_core_cnt = core_cnt; | |
| /* select idle cores that fit all gres binding first */ | |
| if (_handle_core_select(step_ptr, job_resrcs_ptr, | |
| all_gres_core_bitmap, job_node_inx, | |
| sockets, cores, use_all_cores, &core_cnt, | |
| cores_per_task)) | |
| goto cleanup; | |
| /* select idle cores that fit any gres binding second */ | |
| if (!bit_equal(all_gres_core_bitmap, any_gres_core_bitmap) && | |
| _handle_core_select(step_ptr, job_resrcs_ptr, | |
| any_gres_core_bitmap, job_node_inx, | |
| sockets, cores, use_all_cores, &core_cnt, | |
| cores_per_task)) | |
| goto cleanup; | |
| /* select any idle cores */ | |
| if (!(step_ptr->job_ptr->bit_flags & GRES_ENFORCE_BIND) && | |
| !bit_equal(any_gres_core_bitmap, job_resrcs_ptr->core_bitmap)) { | |
| log_flag(STEPS, "gres topology sub-optimal for %ps", | |
| &(step_ptr->step_id)); | |
| if (_handle_core_select(step_ptr, job_resrcs_ptr, | |
| job_resrcs_ptr->core_bitmap, | |
| job_node_inx, sockets, cores, | |
| use_all_cores, &core_cnt, | |
| cores_per_task)) | |
| goto cleanup; | |
| } | |
| /* The test for cores==0 is just to avoid CLANG errors. | |
| * It should never happen */ | |
| if (use_all_cores || (cores == 0)) | |
| goto cleanup; | |
| /* | |
| * Overcommit allows more tasks than CPUs within the same step, | |
| * so not finding a core per task is expected. Succeed as long as | |
| * at least one core was allocated on this node. | |
| */ | |
| if ((step_ptr->flags & SSF_OVERCOMMIT) && | |
| (core_cnt < orig_core_cnt)) { | |
| log_flag(STEPS, "%s: %pS overcommitting on node %s with %d cores allocated to the step. %d more cores requested by the step.", | |
| __func__, step_ptr, | |
| node_record_table_ptr[node_inx]->name, | |
| (orig_core_cnt - core_cnt), core_cnt); | |
| goto cleanup; | |
| } | |
| /* Not enough cores available */ | |
| rc = ESLURM_NODES_BUSY; | |
| cleanup: | |
| FREE_NULL_BITMAP(all_gres_core_bitmap); | |
| FREE_NULL_BITMAP(any_gres_core_bitmap); | |
| return rc; | |
| } | |
| static bool _use_one_thread_per_core(step_record_t *step_ptr) | |
| { | |
| job_record_t *job_ptr = step_ptr->job_ptr; | |
| job_resources_t *job_resrcs_ptr = job_ptr->job_resrcs; | |
| if ((step_ptr->threads_per_core == 1) || | |
| ((step_ptr->threads_per_core == NO_VAL16) && | |
| (job_ptr->details->mc_ptr->threads_per_core == 1)) || | |
| (!(job_resrcs_ptr->whole_node & WHOLE_NODE_REQUIRED) && | |
| (slurm_conf.select_type_param & (SELECT_CORE | SELECT_SOCKET)) && | |
| (job_ptr->details && | |
| (job_ptr->details->cpu_bind_type != NO_VAL16) && | |
| (job_ptr->details->cpu_bind_type & | |
| CPU_BIND_ONE_THREAD_PER_CORE)))) | |
| return true; | |
| return false; | |
| } | |
| static void _modify_cpus_alloc_for_tpc(uint16_t cr_type, uint16_t req_tpc, | |
| uint16_t vpus, int *cpus_alloc) | |
| { | |
| xassert(cpus_alloc); | |
| if ((cr_type & (SELECT_CORE | SELECT_SOCKET | SELECT_LINEAR)) && | |
| (req_tpc != NO_VAL16) && (req_tpc < vpus)) { | |
| *cpus_alloc = ROUNDUP(*cpus_alloc, req_tpc); | |
| *cpus_alloc *= vpus; | |
| } | |
| } | |
| static void _convert_step_cpus_to_job_cpus(uint16_t cr_type, uint16_t req_tpc, | |
| uint16_t job_tpc, uint16_t vpus, | |
| int *core_alloc, int *cpus_alloc) | |
| { | |
| xassert(core_alloc && cpus_alloc); | |
| /* SELECT_CPU's job_resrcs->cpus is based on job_tpc */ | |
| if ((cr_type & SELECT_CPU) && (req_tpc != NO_VAL16)) { | |
| *core_alloc = ROUNDUP(*cpus_alloc, req_tpc); | |
| *cpus_alloc = *core_alloc * | |
| ((job_tpc != NO_VAL16) ? job_tpc : vpus); | |
| } else { | |
| *core_alloc = ROUNDUP(*cpus_alloc, vpus); | |
| *cpus_alloc = *core_alloc * vpus; | |
| } | |
| } | |
| extern node_rank_order_t *step_node_order(step_record_t *step_ptr, | |
| int *order_cnt, | |
| node_rank_order_t **alloc_order) | |
| { | |
| job_resources_t *job_resrcs_ptr = step_ptr->job_ptr->job_resrcs; | |
| slurm_step_layout_t *step_layout = step_ptr->step_layout; | |
| hostlist_t *step_hl; | |
| *alloc_order = NULL; | |
| if ((step_layout->task_dist & SLURM_DIST_STATE_BASE) != | |
| SLURM_DIST_ARBITRARY) { | |
| *order_cnt = job_resrcs_ptr->nhosts; | |
| return job_resrcs_ptr->order_map; | |
| } | |
| step_hl = hostlist_create(step_layout->node_list); | |
| *alloc_order = | |
| _build_arbitrary_order(job_resrcs_ptr, step_hl, order_cnt); | |
| hostlist_destroy(step_hl); | |
| return *alloc_order; | |
| } | |
| /* | |
| * Build the node order from the step layout's node list, regardless of | |
| * distribution. Used at teardown, where a non-arbitrary step may have outlived | |
| * a node removed from its job: the layout still names it, so counting it keeps | |
| * step_node_inx aligned with tasks[]/memory_allocated[]/cpu_alloc_reps[] while | |
| * job_get_node_inx() returns -1 for the gone node. | |
| */ | |
| extern node_rank_order_t *step_layout_order(step_record_t *step_ptr, | |
| int *order_cnt, | |
| node_rank_order_t **alloc_order) | |
| { | |
| job_resources_t *job_resrcs_ptr = step_ptr->job_ptr->job_resrcs; | |
| hostlist_t *step_hl = hostlist_create(step_ptr->step_layout->node_list); | |
| *alloc_order = | |
| _build_arbitrary_order(job_resrcs_ptr, step_hl, order_cnt); | |
| hostlist_destroy(step_hl); | |
| return *alloc_order; | |
| } | |
| extern node_rank_order_t *job_node_order(job_record_t *job_ptr, int *order_cnt, | |
| node_rank_order_t **alloc_order, | |
| char **node_list) | |
| { | |
| job_resources_t *resrcs = job_ptr->job_resrcs; | |
| hostlist_t *nodes_hl; | |
| xassert(resrcs); | |
| *alloc_order = NULL; | |
| if (!job_ptr->details || !job_ptr->details->req_nodes || | |
| ((job_ptr->details->task_dist & SLURM_DIST_STATE_BASE) != | |
| SLURM_DIST_ARBITRARY)) { | |
| *order_cnt = resrcs->nhosts; | |
| *node_list = | |
| job_resources_node_list_by_rank(resrcs, | |
| resrcs->node_bitmap); | |
| return resrcs->order_map; | |
| } | |
| nodes_hl = hostlist_deduplicate(job_ptr->details->req_nodes); | |
| *alloc_order = _build_arbitrary_order(resrcs, nodes_hl, order_cnt); | |
| *node_list = hostlist_ranged_string_xmalloc(nodes_hl); | |
| hostlist_destroy(nodes_hl); | |
| return *alloc_order; | |
| } | |
| extern void job_emit_node_arrays(job_record_t *job_ptr, char **node_list, | |
| uint16_t **cpus_per_node, | |
| uint32_t **cpu_count_reps, | |
| uint32_t *num_cpu_groups) | |
| { | |
| node_rank_order_t *order_map, *alloc_order = NULL; | |
| int order_cnt; | |
| order_map = | |
| job_node_order(job_ptr, &order_cnt, &alloc_order, node_list); | |
| build_job_resources_rank_cpu_array(job_ptr->job_resrcs, order_map, | |
| order_cnt, cpus_per_node, | |
| cpu_count_reps, num_cpu_groups); | |
| xfree(alloc_order); | |
| } | |
| /* Update a job's record of allocated CPUs when a job step gets scheduled */ | |
| static int _step_alloc_lps(step_record_t *step_ptr, char **err_msg) | |
| { | |
| job_record_t *job_ptr = step_ptr->job_ptr; | |
| job_resources_t *job_resrcs_ptr = job_ptr->job_resrcs; | |
| node_record_t *node_ptr; | |
| slurm_step_layout_t *step_layout = step_ptr->step_layout; | |
| int cpus_alloc, cpus_alloc_mem; | |
| int job_node_inx = -1, step_node_inx = -1; | |
| int order_cnt; | |
| node_rank_order_t *order_map, *alloc_order = NULL; | |
| bool first_step_node = true, pick_step_cores = true; | |
| bool all_job_mem = false; | |
| uint32_t rem_nodes; | |
| int rc = SLURM_SUCCESS, final_rc = SLURM_SUCCESS; | |
| multi_core_data_t *mc_ptr = job_ptr->details->mc_ptr; | |
| uint16_t orig_cpus_per_task = step_ptr->cpus_per_task; | |
| uint16_t *cpus_per_task_array = NULL; | |
| uint16_t *cpus_alloc_pn = NULL; | |
| uint16_t ntasks_per_core = step_ptr->ntasks_per_core; | |
| uint16_t req_tpc = _get_threads_per_core(step_ptr->threads_per_core, | |
| job_ptr); | |
| xassert(job_resrcs_ptr); | |
| xassert(job_resrcs_ptr->cpus); | |
| xassert(job_resrcs_ptr->cpus_used); | |
| if (!step_layout) /* batch step */ | |
| return rc; | |
| if (!bit_set_count(job_resrcs_ptr->node_bitmap)) | |
| return rc; | |
| order_map = step_node_order(step_ptr, &order_cnt, &alloc_order); | |
| xfree(*err_msg); | |
| xassert(job_resrcs_ptr->core_bitmap); | |
| xassert(job_resrcs_ptr->core_bitmap_used); | |
| if (step_ptr->core_bitmap_job) { | |
| /* "scontrol reconfig" of live system */ | |
| pick_step_cores = false; | |
| } else if (!(step_ptr->flags & SSF_OVERCOMMIT) && | |
| (step_ptr->cpu_count == job_ptr->total_cpus) && | |
| ((ntasks_per_core == mc_ptr->threads_per_core) || | |
| (ntasks_per_core == INFINITE16))) { | |
| /* | |
| * If the step isn't overcommitting and uses all of job's cores | |
| * Just copy the bitmap to save time | |
| */ | |
| step_ptr->core_bitmap_job = bit_copy( | |
| job_resrcs_ptr->core_bitmap); | |
| pick_step_cores = false; | |
| } | |
| if (step_ptr->pn_min_memory && _is_mem_resv() && | |
| ((job_resrcs_ptr->memory_allocated == NULL) || | |
| (job_resrcs_ptr->memory_used == NULL))) { | |
| error("%s: lack memory allocation details to enforce memory limits for %pJ", | |
| __func__, job_ptr); | |
| step_ptr->pn_min_memory = 0; | |
| } | |
| if (!step_ptr->pn_min_memory) | |
| all_job_mem = true; | |
| rem_nodes = bit_set_count(step_ptr->step_node_bitmap); | |
| xassert(rem_nodes == step_layout->node_cnt); | |
| cpus_alloc_pn = xcalloc(step_layout->node_cnt, sizeof(*cpus_alloc_pn)); | |
| step_ptr->memory_allocated = xcalloc(rem_nodes, sizeof(uint64_t)); | |
| for (int k = 0; k < order_cnt; k++) { | |
| int i = order_map[k].node_inx; | |
| /* | |
| * gres_cpus_alloc - if cpus_per_gres is requested, this is | |
| * cpus_per_gres * gres_alloc on this node | |
| */ | |
| int gres_cpus_alloc = 0; | |
| uint16_t cpus_per_task = orig_cpus_per_task; | |
| uint64_t gres_step_node_mem_alloc = 0; | |
| uint16_t vpus, avail_cpus_per_core, alloc_cpus_per_core; | |
| uint16_t task_cnt; | |
| bitstr_t *unused_core_bitmap; | |
| node_ptr = node_record_table_ptr[i]; | |
| job_node_inx = order_map[k].job_pos; | |
| if (!bit_test(step_ptr->step_node_bitmap, i)) | |
| continue; | |
| step_node_inx++; | |
| if (job_node_inx >= job_resrcs_ptr->nhosts) | |
| fatal("%s: node index bad", __func__); | |
| if (!(task_cnt = step_layout->tasks[step_node_inx])) { | |
| /* This should have been caught earlier */ | |
| error("Bad step layout: no tasks placed on node %d (%s)", | |
| job_node_inx, | |
| node_ptr->name); | |
| final_rc = ESLURM_BAD_TASK_COUNT; | |
| /* | |
| * Finish allocating resources to all nodes to avoid | |
| * underflow errors in _step_alloc_lps | |
| */ | |
| continue; | |
| } | |
| /* | |
| * NOTE: The --overcommit option can result in | |
| * cpus_used[] having a higher value than cpus[] | |
| */ | |
| /* | |
| * If whole allocate all cpus here instead of just the ones | |
| * requested | |
| */ | |
| if (first_step_node) | |
| step_ptr->cpu_count = 0; | |
| vpus = node_ptr->tpc; | |
| if (req_tpc != NO_VAL16) | |
| avail_cpus_per_core = req_tpc; | |
| else | |
| avail_cpus_per_core = vpus; | |
| /* | |
| * If the step requested cpus_per_gres, this is mutually | |
| * exclusive with cpus_per_task. We need to calculate total | |
| * gres times cpus_per_gres to get a total cpu count. | |
| */ | |
| unused_core_bitmap = bit_copy(job_resrcs_ptr->core_bitmap); | |
| bit_and_not(unused_core_bitmap, | |
| job_resrcs_ptr->core_bitmap_used); | |
| rc = gres_stepmgr_step_alloc(step_ptr->gres_list_req, | |
| &step_ptr->gres_list_alloc, | |
| job_ptr->gres_list_alloc, | |
| job_node_inx, first_step_node, | |
| task_cnt, rem_nodes, job_ptr, | |
| step_ptr->step_id.step_id, | |
| !(step_ptr->flags & | |
| SSF_OVERLAP_FORCE), | |
| &gres_step_node_mem_alloc, | |
| node_ptr->gres_list, | |
| unused_core_bitmap, | |
| &gres_cpus_alloc); | |
| FREE_NULL_BITMAP(unused_core_bitmap); | |
| if (rc != SLURM_SUCCESS) { | |
| log_flag(STEPS, "unable to allocate step GRES for job node %d (%s): %s", | |
| job_node_inx, | |
| node_ptr->name, | |
| slurm_strerror(rc)); | |
| /* | |
| * We need to set alloc resources before we continue to | |
| * avoid underflow in _step_dealloc_lps() | |
| */ | |
| final_rc = rc; | |
| } | |
| first_step_node = false; | |
| rem_nodes--; | |
| if (gres_cpus_alloc) { | |
| if (task_cnt > gres_cpus_alloc) { | |
| /* | |
| * Do not error here. If a job requests fewer | |
| * cpus than tasks via cpus_per_gres, | |
| * the job will be allocated one cpu per task. | |
| * Do the same here. | |
| * Use this same logic in _step_dealloc_lps. | |
| */ | |
| cpus_per_task = 1; | |
| log_flag(STEPS, "%s: %pS node %d (%s) gres_cpus_alloc (%d) < tasks (%u), changing gres_cpus_alloc to tasks.", | |
| __func__, step_ptr, job_node_inx, | |
| node_ptr->name, gres_cpus_alloc, | |
| task_cnt); | |
| gres_cpus_alloc = task_cnt; | |
| } else { | |
| cpus_per_task = gres_cpus_alloc / task_cnt; | |
| } | |
| } | |
| /* | |
| * Modify cpus-per-task to request full cores if they can't | |
| * be shared | |
| */ | |
| if ((ntasks_per_core != INFINITE16) && ntasks_per_core) { | |
| alloc_cpus_per_core = avail_cpus_per_core / | |
| ntasks_per_core; | |
| if ((alloc_cpus_per_core > 1) && | |
| (cpus_per_task % alloc_cpus_per_core)) { | |
| cpus_per_task += alloc_cpus_per_core - | |
| (cpus_per_task % alloc_cpus_per_core); | |
| /* | |
| * Modify gres_cpus_alloc to account for | |
| * ntasks_per_core. If this results in | |
| * requesting more cores than are available, | |
| * then _pick_step_cores() will fail. | |
| * | |
| * Make sure to use this same logic in | |
| * _step_dealloc_lps() to know how many | |
| * cpus were allocated to this step on this | |
| * node. | |
| */ | |
| if (gres_cpus_alloc) | |
| gres_cpus_alloc = task_cnt * | |
| cpus_per_task; | |
| } | |
| } | |
| step_ptr->cpus_per_task = cpus_per_task; | |
| /* | |
| * Only populate cpus_per_task_array if needed: if cpus_per_tres | |
| * was requested, then cpus_per_task may not be the same on all | |
| * nodes. Otherwise, cpus_per_task is the same on all nodes, | |
| * and this per-node array isn't needed. | |
| */ | |
| if (gres_cpus_alloc) { | |
| if (!cpus_per_task_array) | |
| cpus_per_task_array = | |
| xcalloc(step_layout->node_cnt, | |
| sizeof(*cpus_per_task_array)); | |
| cpus_per_task_array[step_node_inx] = cpus_per_task; | |
| } | |
| log_flag(STEPS, "%s: %pS node %d (%s) gres_cpus_alloc=%d tasks=%u cpus_per_task=%u", | |
| __func__, step_ptr, job_node_inx, node_ptr->name, | |
| gres_cpus_alloc, task_cnt, | |
| cpus_per_task); | |
| if (step_ptr->flags & SSF_WHOLE) { | |
| cpus_alloc_mem = cpus_alloc = | |
| job_resrcs_ptr->cpus[job_node_inx]; | |
| /* | |
| * If we are requesting all the memory in the job | |
| * (--mem=0) we get it all, otherwise we use what was | |
| * requested specifically for the step. | |
| * | |
| * Else factor in the tpc so we get the correct amount | |
| * of memory. | |
| */ | |
| if (all_job_mem) | |
| cpus_alloc_mem = job_resources_get_node_cpu_cnt( | |
| job_resrcs_ptr, job_node_inx, i); | |
| else if ((req_tpc != NO_VAL16) && | |
| (req_tpc < vpus)) { | |
| cpus_alloc_mem = ROUNDUP(cpus_alloc_mem, vpus); | |
| cpus_alloc_mem *= req_tpc; | |
| } | |
| } else { | |
| if (gres_cpus_alloc) | |
| cpus_alloc = gres_cpus_alloc; | |
| else | |
| cpus_alloc = task_cnt * cpus_per_task; | |
| /* | |
| * If we are requesting all the memory in the job | |
| * (--mem=0) we get it all, otherwise we use what was | |
| * requested specifically for the step. | |
| */ | |
| if (all_job_mem) | |
| cpus_alloc_mem = job_resources_get_node_cpu_cnt( | |
| job_resrcs_ptr, job_node_inx, i); | |
| else | |
| cpus_alloc_mem = cpus_alloc; | |
| /* | |
| * If we are doing threads per core we need the whole | |
| * core allocated even though we are only using what was | |
| * requested. Don't worry about cpus_alloc_mem, it's | |
| * already correct. | |
| */ | |
| _modify_cpus_alloc_for_tpc(job_resrcs_ptr->cr_type, | |
| req_tpc, vpus, &cpus_alloc); | |
| /* | |
| * TODO: We need ntasks-per-* sent to the ctld to make | |
| * more decisions on allocation cores. | |
| */ | |
| } | |
| step_ptr->cpu_count += cpus_alloc; | |
| cpus_alloc_pn[step_node_inx] = cpus_alloc; | |
| /* | |
| * Don't count this step against the allocation if | |
| * --overlap=force | |
| */ | |
| if (!(step_ptr->flags & SSF_OVERLAP_FORCE)) { | |
| int core_alloc; | |
| _convert_step_cpus_to_job_cpus( | |
| job_resrcs_ptr->cr_type, req_tpc, | |
| job_ptr->details->mc_ptr->threads_per_core, | |
| vpus, &core_alloc, &cpus_alloc); | |
| if ((job_resrcs_ptr->cr_type & SELECT_CPU) && | |
| (vpus > 1) && | |
| (job_resrcs_ptr->cpus_used[job_node_inx] + | |
| cpus_alloc) > job_resrcs_ptr->cpus[job_node_inx]) | |
| job_resrcs_ptr->cpus_used[job_node_inx] = | |
| job_resrcs_ptr->cpus[job_node_inx]; | |
| else | |
| job_resrcs_ptr->cpus_used[job_node_inx] += | |
| cpus_alloc; | |
| } | |
| if (!step_ptr->pn_min_memory && !gres_step_node_mem_alloc) { | |
| /* If we aren't requesting memory get it from the job */ | |
| step_ptr->pn_min_memory = | |
| job_resrcs_ptr->memory_allocated[job_node_inx]; | |
| step_ptr->flags |= SSF_MEM_ZERO; | |
| } | |
| if (step_ptr->pn_min_memory && _is_mem_resv()) { | |
| uint64_t mem_use; | |
| if (step_ptr->pn_min_memory & MEM_PER_CPU) { | |
| mem_use = step_ptr->pn_min_memory; | |
| mem_use &= (~MEM_PER_CPU); | |
| mem_use *= cpus_alloc_mem; | |
| } else if (step_ptr->flags & SSF_MEM_ZERO) { | |
| mem_use = job_resrcs_ptr-> | |
| memory_allocated[job_node_inx]; | |
| } else { | |
| mem_use = step_ptr->pn_min_memory; | |
| } | |
| step_ptr->memory_allocated[step_node_inx] = mem_use; | |
| /* | |
| * Do not count against the job's memory allocation if | |
| * --mem=0 or --overlap=force were requested. | |
| */ | |
| if (!(step_ptr->flags & SSF_MEM_ZERO) && | |
| !(step_ptr->flags & SSF_OVERLAP_FORCE)) | |
| job_resrcs_ptr->memory_used[job_node_inx] += | |
| mem_use; | |
| } else if (_is_mem_resv()) { | |
| step_ptr->memory_allocated[step_node_inx] = | |
| gres_step_node_mem_alloc; | |
| /* | |
| * Don't count this step against the allocation if | |
| * --overlap=force | |
| */ | |
| if (!(step_ptr->flags & SSF_OVERLAP_FORCE)) | |
| job_resrcs_ptr->memory_used[job_node_inx] += | |
| gres_step_node_mem_alloc; | |
| } | |
| /* | |
| * Now that we have set cpus and memory used for this node, | |
| * we can check if there was an error, and continue to the | |
| * next node. If any node had an error, we can also skip | |
| * picking cores and skip to the next node. | |
| * | |
| */ | |
| if (final_rc != SLURM_SUCCESS) { | |
| continue; | |
| } | |
| if (pick_step_cores) { | |
| uint16_t cpus_per_core = 1; | |
| /* | |
| * Here we're setting number of CPUs per core | |
| * if we don't enforce 1 thread per core | |
| * | |
| * TODO: move cpus_per_core to slurm_step_layout_t | |
| */ | |
| if (!_use_one_thread_per_core(step_ptr) && | |
| (!(node_ptr->cpus == node_ptr->tot_cores))) { | |
| if (step_ptr->threads_per_core != NO_VAL16) | |
| cpus_per_core = | |
| step_ptr->threads_per_core; | |
| else if (mc_ptr->threads_per_core != NO_VAL16) | |
| cpus_per_core = | |
| mc_ptr->threads_per_core; | |
| else { | |
| cpus_per_core = node_ptr->threads; | |
| } | |
| } | |
| if ((rc = _pick_step_cores(step_ptr, job_resrcs_ptr, | |
| job_node_inx, | |
| task_cnt, | |
| cpus_per_core, i, | |
| ntasks_per_core, | |
| gres_cpus_alloc))) { | |
| log_flag(STEPS, "unable to pick step cores for job node %d (%s): %s", | |
| job_node_inx, | |
| node_ptr->name, | |
| slurm_strerror(rc)); | |
| final_rc = rc; | |
| /* Finish allocating resources to all nodes */ | |
| continue; | |
| } | |
| } | |
| if (slurm_conf.debug_flags & DEBUG_FLAG_CPU_BIND) | |
| _dump_step_layout(step_ptr); | |
| if (step_ptr->flags & SSF_OVERLAP_FORCE) | |
| log_flag(STEPS, "step alloc on job node %d (%s); does not count against job allocation", | |
| job_node_inx, | |
| node_ptr->name); | |
| else | |
| log_flag(STEPS, "step alloc on job node %d (%s) used %u of %u CPUs", | |
| job_node_inx, | |
| node_ptr->name, | |
| job_resrcs_ptr->cpus_used[job_node_inx], | |
| job_resrcs_ptr->cpus[job_node_inx]); | |
| if (step_node_inx == (step_layout->node_cnt - 1)) | |
| break; | |
| } | |
| xfree(alloc_order); | |
| slurm_array16_to_value_reps(cpus_per_task_array, step_layout->node_cnt, | |
| &step_layout->cpt_compact_array, | |
| &step_layout->cpt_compact_reps, | |
| &step_layout->cpt_compact_cnt); | |
| xfree(cpus_per_task_array); | |
| slurm_array16_to_value_reps(cpus_alloc_pn, step_layout->node_cnt, | |
| &step_ptr->cpu_alloc_values, | |
| &step_ptr->cpu_alloc_reps, | |
| &step_ptr->cpu_alloc_array_cnt); | |
| xfree(cpus_alloc_pn); | |
| gres_step_state_log(step_ptr->gres_list_req, job_ptr->job_id, | |
| step_ptr->step_id.step_id); | |
| if ((slurm_conf.debug_flags & DEBUG_FLAG_GRES) && | |
| step_ptr->gres_list_alloc) | |
| info("Step Alloc GRES:"); | |
| gres_step_state_log(step_ptr->gres_list_alloc, job_ptr->job_id, | |
| step_ptr->step_id.step_id); | |
| /* | |
| * If we failed to allocate resources on at least one of the nodes, we | |
| * need to deallocate resources. | |
| * Creating a backup of the resources then restoring in case of an | |
| * error does not work - this method leaves cpus allocated to the node | |
| * after the job completes. Instead, we try to allocate resources on | |
| * all nodes in the job even if one of the nodes resulted in a failure. | |
| */ | |
| if (final_rc != SLURM_SUCCESS) | |
| _step_dealloc_lps(step_ptr); | |
| return final_rc; | |
| } | |
| /* Dump a job step's CPU binding information. | |
| * NOTE: The core_bitmap_job and node index are based upon | |
| * the _job_ allocation */ | |
| static void _dump_step_layout(step_record_t *step_ptr) | |
| { | |
| job_record_t *job_ptr = step_ptr->job_ptr; | |
| job_resources_t *job_resrcs_ptr = job_ptr->job_resrcs; | |
| int i, bit_inx, core_inx, node_inx, rep, sock_inx; | |
| if ((step_ptr->core_bitmap_job == NULL) || | |
| (job_resrcs_ptr == NULL) || | |
| (job_resrcs_ptr->cores_per_socket == NULL)) | |
| return; | |
| info("===================="); | |
| info("%pS", step_ptr); | |
| for (i=0, bit_inx=0, node_inx=0; node_inx<job_resrcs_ptr->nhosts; i++) { | |
| for (rep=0; rep<job_resrcs_ptr->sock_core_rep_count[i]; rep++) { | |
| for (sock_inx=0; | |
| sock_inx<job_resrcs_ptr->sockets_per_node[i]; | |
| sock_inx++) { | |
| for (core_inx=0; | |
| core_inx<job_resrcs_ptr->cores_per_socket[i]; | |
| core_inx++) { | |
| if (bit_test(step_ptr-> | |
| core_bitmap_job, | |
| bit_inx++)) { | |
| info("JobNode[%d] Socket[%d] " | |
| "Core[%d] is allocated", | |
| node_inx, sock_inx, | |
| core_inx); | |
| } | |
| } | |
| } | |
| node_inx++; | |
| } | |
| } | |
| info("===================="); | |
| } | |
| static void _step_dealloc_lps(step_record_t *step_ptr) | |
| { | |
| job_record_t *job_ptr = step_ptr->job_ptr; | |
| job_resources_t *job_resrcs_ptr = job_ptr->job_resrcs; | |
| int cpus_alloc; | |
| int core_alloc; | |
| int job_node_inx = -1, step_node_inx = -1; | |
| int order_cnt; | |
| uint32_t step_id = step_ptr->step_id.step_id; | |
| node_record_t *node_ptr; | |
| node_rank_order_t *order_map, *alloc_order = NULL; | |
| uint16_t job_tpc = job_ptr->details->mc_ptr->threads_per_core; | |
| uint16_t req_tpc = | |
| _get_threads_per_core(step_ptr->threads_per_core, job_ptr); | |
| xassert(job_resrcs_ptr); | |
| if (!job_resrcs_ptr) { | |
| error("%s: job_resrcs is NULL for %pS; this should never happen", | |
| __func__, step_ptr); | |
| return; | |
| } | |
| xassert(job_resrcs_ptr->cpus); | |
| xassert(job_resrcs_ptr->cpus_used); | |
| /* These special steps do not allocate any resources */ | |
| if ((step_id == SLURM_EXTERN_CONT) || | |
| (step_id == SLURM_BATCH_SCRIPT) || | |
| (step_id == SLURM_INTERACTIVE_STEP) || | |
| (step_ptr->flags & SSF_EXT_LAUNCHER)) { | |
| log_flag(STEPS, "Skip %s for %pS", __func__, step_ptr); | |
| return; | |
| } | |
| if (!bit_set_count(job_resrcs_ptr->node_bitmap)) | |
| return; | |
| order_map = step_layout_order(step_ptr, &order_cnt, &alloc_order); | |
| if (step_ptr->memory_allocated && _is_mem_resv() && | |
| ((job_resrcs_ptr->memory_allocated == NULL) || | |
| (job_resrcs_ptr->memory_used == NULL))) { | |
| error("%s: lack memory allocation details to enforce memory limits for %pJ", | |
| __func__, job_ptr); | |
| } | |
| for (int k = 0; k < order_cnt; k++) { | |
| int i = order_map[k].node_inx; | |
| int inx; | |
| uint16_t vpus; | |
| job_node_inx = order_map[k].job_pos; | |
| step_node_inx++; | |
| /* | |
| * The node left the job (shrink, failure) or the cluster; | |
| * its job-side resources are already gone. | |
| */ | |
| if (job_node_inx < 0) | |
| continue; | |
| if (job_node_inx >= job_resrcs_ptr->nhosts) | |
| fatal("_step_dealloc_lps: node index bad"); | |
| node_ptr = node_record_table_ptr[i]; | |
| vpus = node_ptr->tpc; | |
| /* | |
| * We need to free GRES structures regardless of overlap. | |
| */ | |
| gres_stepmgr_step_dealloc(step_ptr->gres_list_alloc, | |
| job_ptr->gres_list_alloc, job_ptr->job_id, | |
| step_ptr->step_id.step_id, | |
| job_node_inx, | |
| !(step_ptr->flags & SSF_OVERLAP_FORCE)); | |
| if (step_ptr->flags & SSF_OVERLAP_FORCE) { | |
| log_flag(STEPS, "step dealloc on job node %d (%s); did not count against job allocation", | |
| job_node_inx, | |
| node_ptr->name); | |
| continue; /* Next node */ | |
| } | |
| /* | |
| * If zero tasks, then _step_alloc_lps() error'd and did not | |
| * allocate any resources, so we should not deallocate anything. | |
| */ | |
| if (!step_ptr->step_layout->tasks[step_node_inx]) | |
| continue; | |
| xassert(step_ptr->cpu_alloc_array_cnt); | |
| xassert(step_ptr->cpu_alloc_reps); | |
| xassert(step_ptr->cpu_alloc_values); | |
| inx = slurm_get_rep_count_inx( | |
| step_ptr->cpu_alloc_reps, | |
| step_ptr->cpu_alloc_array_cnt, | |
| step_node_inx); | |
| cpus_alloc = step_ptr->cpu_alloc_values[inx]; | |
| _convert_step_cpus_to_job_cpus(job_resrcs_ptr->cr_type, req_tpc, | |
| job_tpc, vpus, &core_alloc, | |
| &cpus_alloc); | |
| if ((job_resrcs_ptr->cr_type & SELECT_CPU) && | |
| (node_ptr->tpc > 1)) { | |
| /* SELECT_CPU's job_resrcs->cpus is based on job_tpc */ | |
| uint16_t tpc = (job_tpc != NO_VAL16) ? job_tpc : vpus; | |
| int used_cores = | |
| ROUNDUP(job_resrcs_ptr->cpus_used[job_node_inx], | |
| tpc); | |
| /* | |
| * If SELECT_CPU is used with a thread count > 1 the | |
| * cpus recorded being allocated to a job don't have to | |
| * be a multiple of threads per core. Make sure to | |
| * dealloc full cores and not partial cores. | |
| */ | |
| if (used_cores >= core_alloc) { | |
| used_cores -= core_alloc; | |
| job_resrcs_ptr->cpus_used[job_node_inx] = | |
| MIN(used_cores * tpc, | |
| job_resrcs_ptr->cpus[job_node_inx]); | |
| } else { | |
| error("%s: CPU underflow for %pS (%u<%u on job node %d)", | |
| __func__, step_ptr, used_cores * tpc, | |
| core_alloc * tpc, job_node_inx); | |
| job_resrcs_ptr->cpus_used[job_node_inx] = 0; | |
| } | |
| } else if (job_resrcs_ptr->cpus_used[job_node_inx] >= | |
| cpus_alloc) { | |
| job_resrcs_ptr->cpus_used[job_node_inx] -= cpus_alloc; | |
| } else { | |
| error("%s: CPU underflow for %pS (%u<%u on job node %d)", | |
| __func__, step_ptr, | |
| job_resrcs_ptr->cpus_used[job_node_inx], | |
| cpus_alloc, job_node_inx); | |
| job_resrcs_ptr->cpus_used[job_node_inx] = 0; | |
| } | |
| if (step_ptr->memory_allocated && _is_mem_resv() && | |
| !(step_ptr->flags & SSF_MEM_ZERO)) { | |
| uint64_t mem_use = | |
| step_ptr->memory_allocated[step_node_inx]; | |
| if (job_resrcs_ptr->memory_used[job_node_inx] >= | |
| mem_use) { | |
| job_resrcs_ptr->memory_used[job_node_inx] -= | |
| mem_use; | |
| log_flag(STEPS, "Deallocating %"PRIu64"MB of memory on node %d (%s) now used: %"PRIu64" of %"PRIu64, | |
| mem_use, | |
| job_node_inx, | |
| node_ptr->name, | |
| job_resrcs_ptr-> | |
| memory_used[job_node_inx], | |
| job_resrcs_ptr-> | |
| memory_allocated[job_node_inx]); | |
| } else { | |
| error("%s: Allocated memory underflow for %pS (freed memory=%"PRIu64")", | |
| __func__, step_ptr, mem_use); | |
| job_resrcs_ptr->memory_used[job_node_inx] = 0; | |
| } | |
| } | |
| log_flag(STEPS, "step dealloc on job node %d (%s) used: %u of %u CPUs", | |
| job_node_inx, node_ptr->name, | |
| job_resrcs_ptr->cpus_used[job_node_inx], | |
| job_resrcs_ptr->cpus[job_node_inx]); | |
| if (step_node_inx == (step_ptr->step_layout->node_cnt - 1)) | |
| break; | |
| } | |
| xfree(alloc_order); | |
| xassert(job_resrcs_ptr->core_bitmap); | |
| xassert(job_resrcs_ptr->core_bitmap_used); | |
| if (step_ptr->core_bitmap_job) { | |
| /* Mark the job's cores as no longer in use */ | |
| int job_core_size, step_core_size; | |
| job_core_size = bit_size(job_resrcs_ptr->core_bitmap_used); | |
| step_core_size = bit_size(step_ptr->core_bitmap_job); | |
| /* | |
| * Don't remove step's used cores from job core_bitmap_used if | |
| * SSF_OVERLAP_FORCE | |
| */ | |
| if (job_core_size == step_core_size) { | |
| if (!(step_ptr->flags & SSF_OVERLAP_FORCE)) | |
| bit_and_not(job_resrcs_ptr->core_bitmap_used, | |
| step_ptr->core_bitmap_job); | |
| } else { | |
| error("%s: %pS core_bitmap size mismatch (%d != %d)", | |
| __func__, step_ptr, job_core_size, | |
| step_core_size); | |
| } | |
| FREE_NULL_BITMAP(step_ptr->core_bitmap_job); | |
| } | |
| } | |
| static int _test_strlen(char *test_str, char *str_name, int max_str_len) | |
| { | |
| int i = 0; | |
| if (test_str) | |
| i = strlen(test_str); | |
| if (i > max_str_len) { | |
| info("step_create_request: strlen(%s) too big (%d > %d)", | |
| str_name, i, max_str_len); | |
| return ESLURM_PATHNAME_TOO_LONG; | |
| } | |
| return SLURM_SUCCESS; | |
| } | |
| /* Calculate a step's cpus_per_task value. Set to zero if we can't distributed | |
| * the tasks evenly over the nodes (heterogeneous job allocation). */ | |
| static int _calc_cpus_per_task(step_create_req_msg_t *step_specs, | |
| job_record_t *job_ptr) | |
| { | |
| int cpus_per_task = 0, i; | |
| int num_tasks; | |
| char *cpt = NULL; | |
| if ((cpt = xstrstr(step_specs->tres_per_task, "cpu:"))) { | |
| cpus_per_task = slurm_atoul(cpt + 4); | |
| if (cpus_per_task < 0) | |
| cpus_per_task = 0; | |
| return cpus_per_task; | |
| } | |
| if (step_specs->cpus_per_tres) | |
| return 0; | |
| if (step_specs->num_tasks == NO_VAL) | |
| return 0; | |
| if ((step_specs->cpu_count == 0) || | |
| (step_specs->cpu_count % step_specs->num_tasks)) | |
| return cpus_per_task; | |
| cpus_per_task = step_specs->cpu_count / step_specs->num_tasks; | |
| if (cpus_per_task < 1) | |
| cpus_per_task = 1; | |
| if (!job_ptr->job_resrcs) | |
| return cpus_per_task; | |
| num_tasks = step_specs->num_tasks; | |
| for (i = 0; i < job_ptr->job_resrcs->cpu_array_cnt; i++) { | |
| if (cpus_per_task > job_ptr->job_resrcs->cpu_array_value[i]) { | |
| cpus_per_task = 0; | |
| break; | |
| } | |
| num_tasks -= (job_ptr->job_resrcs->cpu_array_value[i] / | |
| cpus_per_task) * | |
| job_ptr->job_resrcs->cpu_array_reps[i]; | |
| } | |
| if (num_tasks > 0) | |
| return 0; | |
| return cpus_per_task; | |
| } | |
| /* | |
| * Set a job's default cpu_bind_type based upon configuration of allocated nodes, | |
| * partition or global TaskPluginParams | |
| */ | |
| static void _set_def_cpu_bind(job_record_t *job_ptr) | |
| { | |
| job_resources_t *job_resrcs_ptr = job_ptr->job_resrcs; | |
| node_record_t *node_ptr; | |
| uint32_t bind_bits, bind_to_bits, node_bind = NO_VAL; | |
| bool node_fail = false; | |
| if (!job_ptr->details || !job_resrcs_ptr || | |
| !job_resrcs_ptr->node_bitmap) | |
| return; /* No data structure */ | |
| bind_to_bits = CPU_BIND_TO_SOCKETS | CPU_BIND_TO_CORES | | |
| CPU_BIND_TO_THREADS | CPU_BIND_TO_LDOMS; | |
| if ((job_ptr->details->cpu_bind_type != NO_VAL16) && | |
| (job_ptr->details->cpu_bind_type & bind_to_bits)) { | |
| if (slurm_conf.debug_flags & DEBUG_FLAG_CPU_BIND) { | |
| char tmp_str[128]; | |
| slurm_sprint_cpu_bind_type( | |
| tmp_str, job_ptr->details->cpu_bind_type); | |
| log_flag(CPU_BIND, "%pJ CpuBind='%s' already set for job/allocation using it as a default for new step.", | |
| job_ptr, tmp_str); | |
| } | |
| return; /* Already set */ | |
| } | |
| bind_bits = job_ptr->details->cpu_bind_type & CPU_BIND_VERBOSE; | |
| /* | |
| * Set job's cpu_bind to the node's cpu_bind if all of the job's | |
| * allocated nodes have the same cpu_bind (or it is not set) | |
| */ | |
| for (int i = 0; | |
| (node_ptr = next_node_bitmap(job_resrcs_ptr->node_bitmap, &i)); | |
| i++) { | |
| if (node_bind == NO_VAL) { | |
| if (node_ptr->cpu_bind != 0) | |
| node_bind = node_ptr->cpu_bind; | |
| } else if ((node_ptr->cpu_bind != 0) && | |
| (node_bind != node_ptr->cpu_bind)) { | |
| node_fail = true; | |
| break; | |
| } | |
| } | |
| if (!node_fail && (node_bind != NO_VAL)) { | |
| job_ptr->details->cpu_bind_type = bind_bits | node_bind; | |
| if (slurm_conf.debug_flags & DEBUG_FLAG_CPU_BIND) { | |
| char tmp_str[128]; | |
| slurm_sprint_cpu_bind_type( | |
| tmp_str, job_ptr->details->cpu_bind_type); | |
| log_flag(CPU_BIND, "%pJ setting default CpuBind to nodes default '%s' for new step.", | |
| job_ptr, tmp_str); | |
| } | |
| return; | |
| } | |
| /* Use partition's cpu_bind (if any) */ | |
| if (job_ptr->part_ptr && job_ptr->part_ptr->cpu_bind) { | |
| job_ptr->details->cpu_bind_type = bind_bits | | |
| job_ptr->part_ptr->cpu_bind; | |
| if (slurm_conf.debug_flags & DEBUG_FLAG_CPU_BIND) { | |
| char tmp_str[128]; | |
| slurm_sprint_cpu_bind_type( | |
| tmp_str, job_ptr->details->cpu_bind_type); | |
| log_flag(CPU_BIND, "%pJ setting default CpuBind to partition default '%s' for new step.", | |
| job_ptr, tmp_str); | |
| } | |
| return; | |
| } | |
| /* Use global default from TaskPluginParams */ | |
| job_ptr->details->cpu_bind_type = bind_bits | | |
| slurm_conf.task_plugin_param; | |
| if (slurm_conf.debug_flags & DEBUG_FLAG_CPU_BIND) { | |
| char tmp_str[128]; | |
| slurm_sprint_cpu_bind_type(tmp_str, | |
| job_ptr->details->cpu_bind_type); | |
| log_flag(CPU_BIND, "%pJ setting default CpuBind to TaskPluginParam '%s' for new step.", | |
| job_ptr, tmp_str); | |
| } | |
| } | |
| /* | |
| * A step may explicitly set a TRES count to zero in order to avoid making use | |
| * of the job's TRES specifications. At this point, clear the records with | |
| * zero counts. | |
| */ | |
| static void _clear_zero_tres(char **tres_spec) | |
| { | |
| char *new_spec = NULL, *new_sep = ""; | |
| char *tmp, *tok, *sep, *end_ptr = NULL, *save_ptr = NULL; | |
| long int cnt; | |
| if (*tres_spec == NULL) | |
| return; | |
| tmp = xstrdup(*tres_spec); | |
| tok = strtok_r(tmp, ",", &save_ptr); | |
| while (tok) { | |
| bool copy_rec = true; | |
| sep = strrchr(tok, ':'); | |
| if (sep) { | |
| cnt = strtoll(sep+1, &end_ptr, 10); | |
| if ((cnt == 0) && (end_ptr[0] == '\0')) | |
| copy_rec = false; | |
| } | |
| if (copy_rec) { | |
| xstrfmtcat(new_spec, "%s%s", new_sep, tok); | |
| new_sep = ","; | |
| } | |
| tok = strtok_r(NULL, ",", &save_ptr); | |
| } | |
| xfree(tmp); | |
| xfree(*tres_spec); | |
| *tres_spec = new_spec; | |
| } | |
| /* | |
| * A step may explicitly request --gres=none in order to avoid making use | |
| * of the job's TRES specifications. At this point, clear all GRES records. | |
| */ | |
| static void _clear_gres_tres(char **tres_spec) | |
| { | |
| char *new_spec = NULL, *new_sep = ""; | |
| char *tmp, *tok, *save_ptr = NULL; | |
| if (*tres_spec == NULL) | |
| return; | |
| tmp = xstrdup(*tres_spec); | |
| tok = strtok_r(tmp, ",", &save_ptr); | |
| while (tok) { | |
| if (xstrncmp(tok, "gres", 4)) { | |
| xstrfmtcat(new_spec, "%s%s", new_sep, tok); | |
| new_sep = ","; | |
| } | |
| tok = strtok_r(NULL, ",", &save_ptr); | |
| } | |
| xfree(tmp); | |
| xfree(*tres_spec); | |
| *tres_spec = new_spec; | |
| } | |
| /* | |
| * If a job step specification does not include any GRES specification, | |
| * then copy those values from the job record. | |
| * Currently we only want to check if the step lacks a "gres" request. | |
| * "tres_per_[step|task]" has "cpu:<count>" in it, so we need to search for | |
| * "gres" in the strings. | |
| */ | |
| static void _copy_job_tres_to_step(step_create_req_msg_t *step_specs, | |
| job_record_t *job_ptr) | |
| { | |
| if (!xstrcasecmp(step_specs->tres_per_node, "NONE")) { | |
| xfree(step_specs->tres_per_node); | |
| _clear_gres_tres(&step_specs->tres_per_step); | |
| _clear_gres_tres(&step_specs->tres_per_socket); | |
| _clear_gres_tres(&step_specs->tres_per_task); | |
| } else if (xstrstr(step_specs->tres_per_step, "gres") || | |
| xstrstr(step_specs->tres_per_node, "gres") || | |
| xstrstr(step_specs->tres_per_socket, "gres") || | |
| xstrstr(step_specs->tres_per_task, "gres")) { | |
| _clear_zero_tres(&step_specs->tres_per_step); | |
| _clear_zero_tres(&step_specs->tres_per_node); | |
| _clear_zero_tres(&step_specs->tres_per_socket); | |
| _clear_zero_tres(&step_specs->tres_per_task); | |
| } else { | |
| xfree(step_specs->tres_per_step); | |
| xfree(step_specs->tres_per_node); | |
| xfree(step_specs->tres_per_socket); | |
| xfree(step_specs->tres_per_task); | |
| step_specs->tres_per_step = xstrdup(job_ptr->tres_per_job); | |
| step_specs->tres_per_node = xstrdup(job_ptr->tres_per_node); | |
| step_specs->tres_per_socket = xstrdup(job_ptr->tres_per_socket); | |
| step_specs->tres_per_task = xstrdup(job_ptr->tres_per_task); | |
| } | |
| } | |
| static int _test_step_desc_fields(step_create_req_msg_t *step_specs) | |
| { | |
| static time_t sched_update = 0; | |
| static int max_submit_line = DEFAULT_MAX_SUBMIT_LINE_SIZE; | |
| if (sched_update != slurm_conf.last_update) { | |
| char *tmp_ptr; | |
| sched_update = slurm_conf.last_update; | |
| if ((tmp_ptr = xstrcasestr(slurm_conf.sched_params, | |
| "max_submit_line_size="))) { | |
| max_submit_line = atoi(tmp_ptr + 21); | |
| } else { | |
| max_submit_line = DEFAULT_MAX_SUBMIT_LINE_SIZE; | |
| } | |
| } | |
| if (_test_strlen(step_specs->host, "host", 1024) || | |
| _test_strlen(step_specs->name, "name", 1024) || | |
| _test_strlen(step_specs->network, "network", 1024) || | |
| _test_strlen(step_specs->submit_line, "submit_line", | |
| max_submit_line)) | |
| return ESLURM_PATHNAME_TOO_LONG; | |
| return SLURM_SUCCESS; | |
| } | |
| /* | |
| * Whether the job is in a state that permits starting a step on it. | |
| * IN job_ptr - job the step would run under | |
| * RET SLURM_SUCCESS, or the error to report to the requester | |
| */ | |
| static int _job_can_run_step(job_record_t *job_ptr) | |
| { | |
| uint32_t over_time_limit = NO_VAL; | |
| if (!job_ptr->details || IS_JOB_SUSPENDED(job_ptr)) | |
| return ESLURM_DISABLED; | |
| if (job_ptr->node_bitmap_rs) | |
| return ESLURM_INVALID_JOB_STATE; | |
| if (IS_JOB_PENDING(job_ptr)) { | |
| /* NOTE: LSF creates a job allocation for batch jobs. | |
| * After the allocation has been made, LSF submits a | |
| * job to run in that allocation (sbatch --jobid= ...). | |
| * If that job is pending either LSF messed up or LSF is | |
| * not being used. We have seen this problem with Moab. */ | |
| return ESLURM_DUPLICATE_JOB_ID; | |
| } | |
| /* Get OverTimeLimit from job's partition if set, or globally. */ | |
| if (job_ptr->part_ptr && | |
| (job_ptr->part_ptr->over_time_limit != NO_VAL16)) | |
| over_time_limit = job_ptr->part_ptr->over_time_limit; | |
| else | |
| over_time_limit = slurm_conf.over_time_limit; | |
| if (over_time_limit == INFINITE16) | |
| over_time_limit = YEAR_MINUTES; | |
| if (IS_JOB_FINISHED(job_ptr) || | |
| (stepmgr_ops->job_ending && stepmgr_ops->job_ending()) || | |
| (((job_ptr->end_time + (over_time_limit * 60)) <= time(NULL)) && | |
| !IS_JOB_CONFIGURING(job_ptr))) | |
| return ESLURM_ALREADY_DONE; | |
| if (job_ptr->details->prolog_running) | |
| return ESLURM_PROLOG_RUNNING; | |
| return SLURM_SUCCESS; | |
| } | |
| static int _switch_setup(step_record_t *step_ptr) | |
| { | |
| xassert(step_ptr); | |
| if (!step_ptr->step_layout) | |
| return SLURM_SUCCESS; | |
| if (switch_g_stepinfo_build(&step_ptr->switch_step, | |
| step_ptr->job_ptr->switch_jobinfo, | |
| step_ptr) < 0) { | |
| return ESLURM_INTERCONNECT_FAILURE; | |
| } | |
| return SLURM_SUCCESS; | |
| } | |
| /* | |
| * Set the PID limit of a step from its request, or from the job when the | |
| * request has none, matching what slurmstepd enforces on the node. | |
| */ | |
| static void _set_step_max_npids(step_record_t *step_ptr, job_record_t *job_ptr, | |
| step_create_req_msg_t *step_specs) | |
| { | |
| if (step_specs->max_npids == NO_VAL) | |
| step_ptr->max_npids = job_ptr->details->max_npids; | |
| else | |
| step_ptr->max_npids = step_specs->max_npids; | |
| } | |
| /* | |
| * _step_create_internal - creates a step_record in step_specs->job_id, sets up | |
| * the according to the step_specs. | |
| * IN msg - authenticated request and negotiated protocol version | |
| * IN job_ptr - job_ptr to create step in | |
| * IN step_specs - job step specifications | |
| * OUT new_step_record - pointer to the new step_record (NULL on error) | |
| * IN from_wake - true when recreating a step from its own pending | |
| * placeholder at wake, which skips the pending-step lookup and requeue | |
| * IN/OUT dep_ctx - accepted placeholder and newly parsed dependency edges | |
| * OUT err_msg - Custom error message to the user, caller to xfree results | |
| * RET - 0 or error code | |
| * NOTE: don't free the returned step_record because that is managed through | |
| * the job. | |
| */ | |
| static int _step_create_internal(slurm_msg_t *msg, job_record_t *job_ptr, | |
| step_create_req_msg_t *step_specs, | |
| step_record_t **new_step_record, | |
| bool from_wake, step_depend_ctx_t *dep_ctx, | |
| char **err_msg) | |
| { | |
| enum job_state_reason dependency_reason = WAIT_NO_REASON; | |
| step_record_t *step_ptr; | |
| bitstr_t *nodeset; | |
| int cpus_per_task, ret_code, i; | |
| uint32_t node_count = 0; | |
| time_t now = time(NULL); | |
| char *arbitrary_nodes = NULL; | |
| uint32_t orig_cpu_count; | |
| list_t *step_gres_list = NULL; | |
| uint32_t task_dist; | |
| uint32_t max_tasks; | |
| bool resv_ports_present = false; | |
| bool async = step_specs->flags & SSF_ASYNC; | |
| *new_step_record = NULL; | |
| xassert(job_ptr); | |
| /* | |
| * NOTE: We have already confirmed the UID originating | |
| * the request is identical with step_specs->user_id | |
| */ | |
| if (!validate_job_owner(msg, job_ptr)) | |
| return ESLURM_ACCESS_DENIED ; | |
| /* Job-ineligible retries only refresh callback metadata. */ | |
| ret_code = _job_can_run_step(job_ptr); | |
| if (ret_code) { | |
| if ((ret_code == ESLURM_PROLOG_RUNNING) || | |
| launch_step_retry_errno(ret_code)) { | |
| step_create_match_t match = { | |
| .step_specs = step_specs, | |
| }; | |
| list_find_first(job_ptr->step_list, | |
| _find_step_for_create, &match); | |
| if (match.pending_step && | |
| (match.pending_step->step_id.step_id == | |
| SLURM_PENDING_STEP)) | |
| match.pending_step->port = step_specs->port; | |
| } | |
| return ret_code; | |
| } | |
| /* | |
| * Synchronous requests are eligible even with a NO_VAL id: a re-send | |
| * that raced the queued response (controller committed a placeholder, | |
| * srun never saw the id) comes back as NO_VAL and must still find its | |
| * own pending placeholder instead of stacking a second one. A | |
| * create-at-wake already has its placeholder and skips this lookup. | |
| */ | |
| if (!from_wake && (!async || ((step_specs->step_id.step_id != NO_VAL) && | |
| !step_specs->immediate))) { | |
| step_create_match_t match = { | |
| .step_specs = step_specs, | |
| }; | |
| if (list_find_first(job_ptr->step_list, _find_step_for_create, | |
| &match) && | |
| !match.pending_step) | |
| return ESLURM_DUPLICATE_STEP_ID; | |
| dep_ctx->pending_step = match.pending_step; | |
| if (dep_ctx->pending_step) { | |
| step_record_t *pending_step = dep_ctx->pending_step; | |
| if ((step_specs->step_id.step_id == NO_VAL) && | |
| (pending_step->step_id.step_id != | |
| SLURM_PENDING_STEP)) | |
| step_specs->step_id.step_id = | |
| pending_step->step_id.step_id; | |
| if (xstrcmp(pending_step->step_dependency, | |
| step_specs->step_dependency)) { | |
| error("%s: re-sent %ps changed its step dependency", | |
| __func__, &step_specs->step_id); | |
| return ESLURM_DEPENDENCY; | |
| } | |
| } | |
| } | |
| if ((step_specs->max_npids != NO_VAL) && | |
| (!step_specs->max_npids || | |
| ((job_ptr->details->max_npids != NO_VAL) && | |
| (step_specs->max_npids > job_ptr->details->max_npids)))) | |
| return ESLURM_INVALID_MAX_PIDS; | |
| /* | |
| * Reject a dependency the special step kinds below cannot honour, | |
| * before their early returns skip the check. | |
| */ | |
| if (step_specs->step_dependency && | |
| (step_specs->flags & (SSF_INTERACTIVE | SSF_EXT_LAUNCHER))) { | |
| error("%s: step dependencies are not supported for interactive or external launcher steps", | |
| __func__); | |
| return ESLURM_DEPENDENCY; | |
| } | |
| if (step_specs->flags & SSF_INTERACTIVE) { | |
| debug("%s: interactive step requested", __func__); | |
| *new_step_record = | |
| _build_interactive_step(job_ptr, step_specs, | |
| msg->protocol_version); | |
| if (*new_step_record) | |
| return SLURM_SUCCESS; | |
| else | |
| return ESLURM_DUPLICATE_STEP_ID; | |
| } | |
| if (step_specs->flags & SSF_EXT_LAUNCHER) { | |
| debug("%s: external launcher step requested", __func__); | |
| return _build_ext_launcher_step(new_step_record, job_ptr, | |
| step_specs, | |
| msg->protocol_version); | |
| } | |
| /* A step cannot request more threads per core than its allocation. */ | |
| if ((step_specs->threads_per_core != NO_VAL16) && | |
| (step_specs->threads_per_core > | |
| job_ptr->job_resrcs->threads_per_core)) | |
| return ESLURM_BAD_THREAD_PER_CORE; | |
| task_dist = step_specs->task_dist & SLURM_DIST_STATE_BASE; | |
| /* Set to block in the case that mem is 0. srun leaves the dist | |
| * set to unknown if mem is 0. */ | |
| if ((task_dist == SLURM_DIST_UNKNOWN) && | |
| (!(step_specs->pn_min_memory &(~MEM_PER_CPU)))) { | |
| step_specs->task_dist &= SLURM_DIST_STATE_FLAGS; | |
| step_specs->task_dist |= SLURM_DIST_BLOCK; | |
| task_dist = SLURM_DIST_BLOCK; | |
| } | |
| if ((task_dist != SLURM_DIST_CYCLIC) && | |
| (task_dist != SLURM_DIST_BLOCK) && | |
| (task_dist != SLURM_DIST_CYCLIC_CYCLIC) && | |
| (task_dist != SLURM_DIST_BLOCK_CYCLIC) && | |
| (task_dist != SLURM_DIST_CYCLIC_BLOCK) && | |
| (task_dist != SLURM_DIST_BLOCK_BLOCK) && | |
| (task_dist != SLURM_DIST_CYCLIC_CFULL) && | |
| (task_dist != SLURM_DIST_BLOCK_CFULL) && | |
| (task_dist != SLURM_DIST_CYCLIC_CYCLIC_CYCLIC) && | |
| (task_dist != SLURM_DIST_CYCLIC_CYCLIC_BLOCK) && | |
| (task_dist != SLURM_DIST_CYCLIC_CYCLIC_CFULL) && | |
| (task_dist != SLURM_DIST_CYCLIC_BLOCK_CYCLIC) && | |
| (task_dist != SLURM_DIST_CYCLIC_BLOCK_BLOCK) && | |
| (task_dist != SLURM_DIST_CYCLIC_BLOCK_CFULL) && | |
| (task_dist != SLURM_DIST_CYCLIC_CFULL_CYCLIC) && | |
| (task_dist != SLURM_DIST_CYCLIC_CFULL_BLOCK) && | |
| (task_dist != SLURM_DIST_CYCLIC_CFULL_CFULL) && | |
| (task_dist != SLURM_DIST_BLOCK_CYCLIC_CYCLIC) && | |
| (task_dist != SLURM_DIST_BLOCK_CYCLIC_BLOCK) && | |
| (task_dist != SLURM_DIST_BLOCK_CYCLIC_CFULL) && | |
| (task_dist != SLURM_DIST_BLOCK_BLOCK_CYCLIC) && | |
| (task_dist != SLURM_DIST_BLOCK_BLOCK_BLOCK) && | |
| (task_dist != SLURM_DIST_BLOCK_BLOCK_CFULL) && | |
| (task_dist != SLURM_DIST_BLOCK_CFULL_CYCLIC) && | |
| (task_dist != SLURM_DIST_BLOCK_CFULL_BLOCK) && | |
| (task_dist != SLURM_DIST_BLOCK_CFULL_CFULL) && | |
| (task_dist != SLURM_DIST_PLANE) && | |
| (task_dist != SLURM_DIST_ARBITRARY)) | |
| return ESLURM_BAD_DIST; | |
| if (!assoc_mgr_valid_tres_cnt(step_specs->cpus_per_tres, 0) || | |
| !assoc_mgr_valid_tres_cnt(step_specs->mem_per_tres, 0) || | |
| tres_bind_verify_cmdline(step_specs->tres_bind) || | |
| tres_freq_verify_cmdline(step_specs->tres_freq) || | |
| !assoc_mgr_valid_tres_cnt(step_specs->tres_per_step, 0) || | |
| (!assoc_mgr_valid_tres_cnt(step_specs->tres_per_node, 0) && | |
| xstrcasecmp(step_specs->tres_per_node, "NONE")) || | |
| !assoc_mgr_valid_tres_cnt(step_specs->tres_per_socket, 0) || | |
| !assoc_mgr_valid_tres_cnt(step_specs->tres_per_task, 0)) | |
| return ESLURM_INVALID_TRES; | |
| if ((ret_code = _test_step_desc_fields(step_specs)) != SLURM_SUCCESS) | |
| return ret_code; | |
| if (job_ptr->next_step_id >= slurm_conf.max_step_cnt) | |
| return ESLURM_STEP_LIMIT; | |
| /* | |
| * Resolve the step's dependencies after the static request checks, so | |
| * an obviously invalid request is still rejected rather than accepted | |
| * as a queued step, but before any resources are picked. | |
| * | |
| * A wake retry already reduced the accepted edges on its placeholder. | |
| * The request string is only parsed on first admission: by the time the | |
| * step runs, targets admitted here may have finished and be gone, and | |
| * re-parsing would reject a dependency that was legitimately satisfied. | |
| */ | |
| if (!from_wake && step_specs->step_dependency) { | |
| list_t *depend_list = NULL; | |
| bool or_flag = false; | |
| if (dep_ctx->pending_step) { | |
| depend_list = dep_ctx->pending_step->step_depend_list; | |
| or_flag = dep_ctx->pending_step->step_depend_or; | |
| } else { | |
| ret_code = | |
| step_depend_parse(job_ptr, &step_specs->step_id, | |
| step_specs->step_dependency, | |
| &dep_ctx->depend_list, | |
| &dep_ctx->or_flag); | |
| if (ret_code) | |
| return ret_code; | |
| depend_list = dep_ctx->depend_list; | |
| or_flag = dep_ctx->or_flag; | |
| } | |
| dep_ctx->expression = step_specs->step_dependency; | |
| dependency_reason = | |
| step_depend_reduce_list(depend_list, or_flag); | |
| } | |
| if (dependency_reason == WAIT_DEPENDENCY) { | |
| /* | |
| * Queue the step without picking any resources for it, even if | |
| * the job is idle: it is waiting on its predecessors, not on | |
| * CPUs. --immediate asked not to wait, so tell it instead. | |
| */ | |
| if (step_specs->immediate && !from_wake) | |
| return ESLURM_STEP_DEP_NOT_MET; | |
| return _queue_pending_step(job_ptr, step_specs, | |
| msg->protocol_version, | |
| ESLURM_STEP_DEP_NOT_MET, from_wake, | |
| dep_ctx); | |
| } | |
| if (dependency_reason == WAIT_DEP_INVALID) { | |
| /* A predecessor ended in a way this dependency rejects. */ | |
| return ESLURM_STEP_DEP_FAILED; | |
| } | |
| /* | |
| * If the overcommit flag is checked, we set cpu_count=0 | |
| * which makes it so we don't check to see the available cpus | |
| */ | |
| orig_cpu_count = step_specs->cpu_count; | |
| if (step_specs->flags & SSF_OVERCOMMIT) | |
| step_specs->cpu_count = 0; | |
| if (!step_specs->ntasks_per_tres) | |
| step_specs->ntasks_per_tres = NO_VAL16; | |
| /* determine cpus_per_task value by reversing what srun does */ | |
| if (step_specs->num_tasks < 1) | |
| return ESLURM_BAD_TASK_COUNT; | |
| cpus_per_task = _calc_cpus_per_task(step_specs, job_ptr); | |
| _copy_job_tres_to_step(step_specs, job_ptr); | |
| /* If whole is given we probably need to copy tres_per_* from the job */ | |
| i = gres_step_state_validate(step_specs->cpus_per_tres, | |
| step_specs->tres_per_step, | |
| step_specs->tres_per_node, | |
| step_specs->tres_per_socket, | |
| step_specs->tres_per_task, | |
| step_specs->mem_per_tres, | |
| step_specs->ntasks_per_tres, | |
| step_specs->min_nodes, | |
| &step_gres_list, | |
| job_ptr->job_id, | |
| NO_VAL, &step_specs->num_tasks, | |
| &step_specs->cpu_count, err_msg); | |
| if (i != SLURM_SUCCESS) { | |
| FREE_NULL_LIST(step_gres_list); | |
| return i; | |
| } | |
| job_ptr->time_last_active = now; | |
| nodeset = _pick_step_nodes(job_ptr, step_specs, step_gres_list, | |
| cpus_per_task, node_count, &ret_code); | |
| if (nodeset == NULL) { | |
| FREE_NULL_LIST(step_gres_list); | |
| if (ret_code == ESLURM_NODES_BUSY) | |
| ret_code = _queue_pending_step(job_ptr, step_specs, | |
| msg->protocol_version, | |
| ret_code, from_wake, | |
| dep_ctx); | |
| return ret_code; | |
| } | |
| _set_def_cpu_bind(job_ptr); | |
| node_count = bit_set_count(nodeset); | |
| xassert(step_specs->num_tasks != NO_VAL); | |
| max_tasks = node_count * slurm_conf.max_tasks_per_node; | |
| if (step_specs->num_tasks > max_tasks) { | |
| error("step has invalid task count: %u max is %u", | |
| step_specs->num_tasks, max_tasks); | |
| FREE_NULL_LIST(step_gres_list); | |
| FREE_NULL_BITMAP(nodeset); | |
| return ESLURM_BAD_TASK_COUNT; | |
| } | |
| step_ptr = create_step_record(job_ptr, msg->protocol_version); | |
| if (step_ptr == NULL) { | |
| FREE_NULL_LIST(step_gres_list); | |
| FREE_NULL_BITMAP(nodeset); | |
| return ESLURMD_TOOMANYSTEPS; | |
| } | |
| *stepmgr_ops->last_job_update = time(NULL); | |
| step_ptr->start_time = time(NULL); | |
| step_ptr->state = JOB_RUNNING; | |
| if (async && step_specs->immediate) { | |
| /* Async pending step already has a step_id. */ | |
| step_ptr->step_id = step_specs->step_id; | |
| } else if (!async) { | |
| if (step_specs->step_id.step_id != NO_VAL) { | |
| /* Sync pending step already has a step_id. */ | |
| step_ptr->step_id = step_specs->step_id; | |
| step_ptr->step_id.sluid = job_ptr->step_id.sluid; | |
| if (step_specs->array_task_id != NO_VAL) | |
| step_ptr->step_id.job_id = job_ptr->job_id; | |
| /* | |
| * Keep the auto-assign counter past a caller-supplied | |
| * id without consuming one, so a re-send of an already | |
| * assigned id does not leave a StepId gap. | |
| */ | |
| if (step_ptr->step_id.step_het_comp == NO_VAL) | |
| job_ptr->next_step_id = | |
| MAX(job_ptr->next_step_id, | |
| step_ptr->step_id.step_id + 1); | |
| } else { | |
| if ((ret_code = _set_step_id(step_ptr, step_specs))) { | |
| delete_step_record(job_ptr, step_ptr); | |
| return ret_code; | |
| } | |
| /* | |
| * Echo the assigned id back so that if this create | |
| * then fails on busy reserved ports the pending | |
| * placeholder reuses it, instead of | |
| * _build_pending_step() assigning a second id and | |
| * skipping a StepId. | |
| */ | |
| step_specs->step_id = step_ptr->step_id; | |
| } | |
| } | |
| /* | |
| * For arbitrary distribution the user-supplied node list drives task | |
| * placement; step_layout_create() builds the node list otherwise. | |
| */ | |
| if (step_specs->node_list && | |
| ((step_specs->task_dist & SLURM_DIST_STATE_BASE) == | |
| SLURM_DIST_ARBITRARY)) | |
| arbitrary_nodes = step_specs->node_list; | |
| step_ptr->step_node_bitmap = nodeset; | |
| step_ptr->container = xstrdup(step_specs->container); | |
| step_ptr->container_id = xstrdup(step_specs->container_id); | |
| step_ptr->runtime = xstrdup(step_specs->runtime); | |
| step_ptr->gres_list_req = step_gres_list; | |
| step_gres_list = NULL; | |
| gres_step_state_log(step_ptr->gres_list_req, job_ptr->job_id, | |
| step_ptr->step_id.step_id); | |
| if ((slurm_conf.debug_flags & DEBUG_FLAG_GRES) && | |
| step_ptr->gres_list_alloc) | |
| info("Step Alloc GRES:"); | |
| gres_step_state_log(step_ptr->gres_list_alloc, job_ptr->job_id, | |
| step_ptr->step_id.step_id); | |
| step_ptr->port = step_specs->port; | |
| step_ptr->srun_pid = step_specs->srun_pid; | |
| step_ptr->host = xstrdup(step_specs->host); | |
| _set_step_tls_cert(step_ptr, step_specs); | |
| if ((step_specs->cpu_freq_min == NO_VAL) && | |
| (step_specs->cpu_freq_max == NO_VAL) && | |
| (step_specs->cpu_freq_gov == NO_VAL)) { | |
| step_ptr->cpu_freq_min = job_ptr->details->cpu_freq_min; | |
| step_ptr->cpu_freq_max = job_ptr->details->cpu_freq_max; | |
| step_ptr->cpu_freq_gov = job_ptr->details->cpu_freq_gov; | |
| } else { | |
| step_ptr->cpu_freq_min = step_specs->cpu_freq_min; | |
| step_ptr->cpu_freq_max = step_specs->cpu_freq_max; | |
| step_ptr->cpu_freq_gov = step_specs->cpu_freq_gov; | |
| } | |
| _set_step_max_npids(step_ptr, job_ptr, step_specs); | |
| step_ptr->cpus_per_task = (uint16_t)cpus_per_task; | |
| step_ptr->ntasks_per_core = step_specs->ntasks_per_core; | |
| step_ptr->pn_min_memory = step_specs->pn_min_memory; | |
| /* | |
| * cpu_count can be updated by gres_step_state_validate() if OVERCOMMIT | |
| * is not used. If so, use the updated value. | |
| */ | |
| if (step_specs->flags & SSF_OVERCOMMIT) | |
| step_ptr->cpu_count = orig_cpu_count; | |
| else | |
| step_ptr->cpu_count = step_specs->cpu_count; | |
| step_ptr->exit_code = NO_VAL; | |
| step_ptr->flags = step_specs->flags; | |
| step_ptr->cpus_per_tres = xstrdup(step_specs->cpus_per_tres); | |
| step_ptr->mem_per_tres = xstrdup(step_specs->mem_per_tres); | |
| step_ptr->cwd = xstrdup(step_specs->cwd); | |
| step_ptr->std_err = xstrdup(step_specs->std_err); | |
| step_ptr->std_in = xstrdup(step_specs->std_in); | |
| step_ptr->std_out = xstrdup(step_specs->std_out); | |
| step_ptr->submit_line = xstrdup(step_specs->submit_line); | |
| step_ptr->tres_bind = xstrdup(step_specs->tres_bind); | |
| step_ptr->tres_freq = xstrdup(step_specs->tres_freq); | |
| step_ptr->tres_per_step = xstrdup(step_specs->tres_per_step); | |
| step_ptr->tres_per_node = xstrdup(step_specs->tres_per_node); | |
| step_ptr->tres_per_socket = xstrdup(step_specs->tres_per_socket); | |
| step_ptr->tres_per_task = xstrdup(step_specs->tres_per_task); | |
| step_ptr->threads_per_core = step_specs->threads_per_core; | |
| /* | |
| * step's name and network default to job's values if not | |
| * specified in the step specification | |
| */ | |
| if (step_specs->name && step_specs->name[0]) | |
| step_ptr->name = xstrdup(step_specs->name); | |
| else | |
| step_ptr->name = xstrdup(job_ptr->name); | |
| if (step_specs->network && step_specs->network[0]) | |
| step_ptr->network = xstrdup(step_specs->network); | |
| else | |
| step_ptr->network = xstrdup(job_ptr->network); | |
| /* | |
| * the step time_limit is recorded as submitted (INFINITE | |
| * or partition->max_time by default), but the allocation | |
| * time limits may cut it short | |
| */ | |
| if (step_specs->time_limit == NO_VAL || step_specs->time_limit == 0 || | |
| step_specs->time_limit == INFINITE) { | |
| step_ptr->time_limit = INFINITE; | |
| } else { | |
| /* enforce partition limits if necessary */ | |
| if ((step_specs->time_limit > job_ptr->part_ptr->max_time) && | |
| slurm_conf.enforce_part_limits) { | |
| info("%s: %pS time greater than partition's (%u > %u)", | |
| __func__, step_ptr, step_specs->time_limit, | |
| job_ptr->part_ptr->max_time); | |
| delete_step_record(job_ptr, step_ptr); | |
| return ESLURM_INVALID_TIME_LIMIT; | |
| } | |
| step_ptr->time_limit = step_specs->time_limit; | |
| } | |
| step_ptr->step_layout = | |
| step_layout_create(step_ptr, arbitrary_nodes, node_count, | |
| step_specs->num_tasks, | |
| (uint16_t) cpus_per_task, | |
| step_specs->task_dist, | |
| step_specs->plane_size); | |
| if (!step_ptr->step_layout) { | |
| delete_step_record(job_ptr, step_ptr); | |
| if (step_specs->pn_min_memory) | |
| return ESLURM_INVALID_TASK_MEMORY; | |
| return SLURM_ERROR; | |
| } | |
| if (slurm_conf.mpi_params && xstrstr(slurm_conf.mpi_params, "ports=")) | |
| resv_ports_present = true; | |
| if ((step_specs->resv_port_cnt == NO_VAL16) && | |
| (resv_ports_present || job_ptr->resv_ports)) { | |
| step_specs->resv_port_cnt = 0; | |
| /* | |
| * reserved port count set to maximum task count on | |
| * any node plus one | |
| */ | |
| for (i = 0; i < step_ptr->step_layout->node_cnt; i++) { | |
| step_specs->resv_port_cnt = | |
| MAX(step_specs->resv_port_cnt, | |
| step_ptr->step_layout->tasks[i]); | |
| } | |
| step_specs->resv_port_cnt++; | |
| } | |
| if ((step_specs->resv_port_cnt != NO_VAL16) && | |
| (step_specs->resv_port_cnt != 0)) { | |
| step_ptr->resv_port_cnt = step_specs->resv_port_cnt; | |
| ret_code = resv_port_step_alloc(step_ptr); | |
| if (ret_code != SLURM_SUCCESS) { | |
| if (ret_code == ESLURM_PORTS_BUSY) | |
| ret_code = _queue_pending_step( | |
| job_ptr, step_specs, | |
| msg->protocol_version, ret_code, | |
| from_wake, dep_ctx); | |
| delete_step_record(job_ptr, step_ptr); | |
| return ret_code; | |
| } | |
| } | |
| if (async && !step_specs->immediate && | |
| (ret_code = _set_step_id(step_ptr, step_specs))) { | |
| delete_step_record(job_ptr, step_ptr); | |
| return ret_code; | |
| } | |
| if ((ret_code = _switch_setup(step_ptr))) { | |
| delete_step_record(job_ptr, step_ptr); | |
| return ret_code; | |
| } | |
| if ((ret_code = _step_alloc_lps(step_ptr, err_msg))) { | |
| delete_step_record(job_ptr, step_ptr); | |
| return ret_code; | |
| } | |
| xassert(bit_set_count(step_ptr->core_bitmap_job) != 0); | |
| /* Keep the expression, but let create cleanup free the spent edges. */ | |
| step_ptr->step_dependency = xstrdup(dep_ctx->expression); | |
| *new_step_record = step_ptr; | |
| step_set_alloc_tres(step_ptr, node_count, false, true); | |
| jobacct_storage_g_step_start(stepmgr_ops->acct_db_conn, step_ptr); | |
| /* Transfer outcome ownership before publishing the committed start. */ | |
| if (dep_ctx->pending_step) { | |
| xassert(!step_ptr->step_depend_outcome); | |
| step_ptr->step_depend_outcome = | |
| dep_ctx->pending_step->step_depend_outcome; | |
| dep_ctx->pending_step->step_depend_outcome = NULL; | |
| } | |
| dep_ctx->dependency_reconsider = _publish_step_outcome(step_ptr, true); | |
| return SLURM_SUCCESS; | |
| } | |
| /* | |
| * Build the node ordering for an arbitrary-distribution step from its | |
| * (already deduplicated) node list, so the per-node arrays built in | |
| * step_layout_create() stay aligned with that list. job_pos is the node's | |
| * position in the job's resources (bitmap order). | |
| */ | |
| static node_rank_order_t *_build_arbitrary_order(job_resources_t | |
| *job_resrcs_ptr, | |
| hostlist_t *dedup_nodes_hl, | |
| int *order_cnt) | |
| { | |
| node_rank_order_t *order; | |
| hostlist_iterator_t *itr = hostlist_iterator_create(dedup_nodes_hl); | |
| char *host; | |
| int cnt = 0; | |
| /* | |
| * The list is validated against the job when the step is created, | |
| * but a node can leave the job afterwards (shrink, node failure), | |
| * so the list can name more nodes than the job now has. Size by the | |
| * list; a node not in the cluster or the job bitmap gets job_pos -1, | |
| * and the consumers skip its job-side accounting while still | |
| * counting it to stay aligned with the step layout. | |
| */ | |
| order = xcalloc(hostlist_count(dedup_nodes_hl), sizeof(*order)); | |
| while ((host = hostlist_next(itr))) { | |
| order[cnt].job_pos = | |
| job_get_node_inx(host, job_resrcs_ptr->node_bitmap); | |
| if (order[cnt].job_pos >= 0) | |
| order[cnt].node_inx = node_name_get_inx(host); | |
| else | |
| order[cnt].node_inx = NO_VAL; | |
| cnt++; | |
| xfree(host); | |
| } | |
| hostlist_iterator_destroy(itr); | |
| *order_cnt = cnt; | |
| return order; | |
| } | |
| /* | |
| * Own new dependencies and retire an accepted placeholder only on success or | |
| * terminal failure. | |
| * IN msg - authenticated request and negotiated protocol version | |
| * IN job_ptr - job to create the step in | |
| * IN/OUT step_specs - job step specifications | |
| * OUT new_step_record - the new step_record (NULL on error) | |
| * IN/OUT wake_step - caller-owned wake placeholder, NULL for RPC creation | |
| * OUT dependency_reconsider - pending dependencies should be reconsidered | |
| * OUT err_msg - custom error message to the user, caller to xfree | |
| * RET SLURM_SUCCESS or an error code | |
| */ | |
| static int _step_create(slurm_msg_t *msg, job_record_t *job_ptr, | |
| step_create_req_msg_t *step_specs, | |
| step_record_t **new_step_record, | |
| step_record_t *wake_step, bool *dependency_reconsider, | |
| char **err_msg) | |
| { | |
| step_depend_ctx_t dep_ctx = { .pending_step = wake_step }; | |
| bool from_wake = !!wake_step; | |
| int rc = EINVAL; | |
| rc = _step_create_internal(msg, job_ptr, step_specs, new_step_record, | |
| from_wake, &dep_ctx, err_msg); | |
| FREE_NULL_LIST(dep_ctx.depend_list); | |
| if (dep_ctx.pending_step && !from_wake) { | |
| if (rc && ((rc == ESLURM_PROLOG_RUNNING) || | |
| launch_step_retry_errno(rc))) { | |
| /* Older srun reopens its listener on each retry. */ | |
| dep_ctx.pending_step->port = step_specs->port; | |
| dep_ctx.pending_step->time_last_active = time(NULL); | |
| } else { | |
| if (rc) { | |
| /* Fail an impossible dependency, as on wake. */ | |
| if (rc == ESLURM_STEP_DEP_FAILED) | |
| _fail_unlaunched_step( | |
| dep_ctx.pending_step); | |
| else | |
| _complete_unlaunched_step( | |
| dep_ctx.pending_step); | |
| dep_ctx.dependency_reconsider |= | |
| _publish_step_outcome( | |
| dep_ctx.pending_step, false); | |
| } | |
| delete_step_record(job_ptr, dep_ctx.pending_step); | |
| } | |
| } | |
| *dependency_reconsider = dep_ctx.dependency_reconsider; | |
| return rc; | |
| } | |
| extern slurm_step_layout_t *step_layout_create(step_record_t *step_ptr, | |
| char *arbitrary_nodes, | |
| uint32_t node_count, | |
| uint32_t num_tasks, | |
| uint16_t cpus_per_task, | |
| uint32_t task_dist, | |
| uint16_t plane_size) | |
| { | |
| slurm_step_layout_t *step_layout = NULL; | |
| uint16_t cpus_per_node[node_count]; | |
| uint16_t cpus_per_task_array[node_count]; | |
| char *step_nodes = NULL; | |
| job_record_t *job_ptr = step_ptr->job_ptr; | |
| job_resources_t *job_resrcs_ptr = job_ptr->job_resrcs; | |
| slurm_step_layout_req_t step_layout_req = { 0 }; | |
| uint64_t gres_cpus; | |
| int usable_cpus, usable_mem; | |
| int set_nodes = 0 /* , set_tasks = 0 */; | |
| uint16_t ntasks_per_core = step_ptr->ntasks_per_core; | |
| node_record_t *node_ptr; | |
| node_rank_order_t *order_map, *arbitrary_order = NULL; | |
| int order_cnt; | |
| hostlist_t *step_hl = NULL; | |
| gres_stepmgr_step_test_args_t gres_test_args = { | |
| .cpus_per_task = step_ptr->cpus_per_task, | |
| .first_step_node = true, | |
| .job_gres_list = job_ptr->gres_list_alloc, | |
| .job_id = job_ptr->job_id, | |
| .job_resrcs_ptr = job_resrcs_ptr, | |
| .node_offset = -1, | |
| .step_gres_list = step_ptr->gres_list_req, | |
| .step_id = step_ptr->step_id.step_id, | |
| .test_mem = false, | |
| }; | |
| xassert(job_resrcs_ptr); | |
| xassert(job_resrcs_ptr->cpus); | |
| xassert(job_resrcs_ptr->cpus_used); | |
| if (step_ptr->pn_min_memory && _is_mem_resv() && | |
| ((job_resrcs_ptr->memory_allocated == NULL) || | |
| (job_resrcs_ptr->memory_used == NULL))) { | |
| error("%s: lack memory allocation details to enforce memory limits for %pJ", | |
| __func__, job_ptr); | |
| step_ptr->pn_min_memory = 0; | |
| } else if (step_ptr->pn_min_memory == MEM_PER_CPU) | |
| step_ptr->pn_min_memory = 0; /* clear MEM_PER_CPU flag */ | |
| /* build cpus-per-node arrays for the subset of nodes used by step */ | |
| gres_test_args.max_rem_nodes = | |
| bit_set_count(step_ptr->step_node_bitmap); | |
| if ((!step_ptr->pn_min_memory) && _is_mem_resv()) { | |
| gres_test_args.test_mem = true; | |
| } | |
| if (step_ptr->flags & SSF_OVERLAP_FORCE) | |
| gres_test_args.ignore_alloc = true; | |
| else | |
| gres_test_args.ignore_alloc = false; | |
| /* | |
| * Emit the step's nodes (and the parallel per-node arrays below) so | |
| * that observable step state matches task placement: topology-rank | |
| * order for regular distributions, and the user's first-occurrence | |
| * order for arbitrary. step_nodes holds the resulting node list. | |
| */ | |
| if (arbitrary_nodes) { | |
| hostlist_t *hl = hostlist_deduplicate(arbitrary_nodes); | |
| arbitrary_order = | |
| _build_arbitrary_order(job_resrcs_ptr, hl, &order_cnt); | |
| step_nodes = hostlist_ranged_string_xmalloc(hl); | |
| hostlist_destroy(hl); | |
| order_map = arbitrary_order; | |
| } else { | |
| order_map = job_resrcs_ptr->order_map; | |
| order_cnt = job_resrcs_ptr->nhosts; | |
| step_hl = hostlist_create(NULL); | |
| } | |
| for (int k = 0; k < order_cnt; k++) { | |
| int i = order_map[k].node_inx; | |
| int pos = order_map[k].job_pos; | |
| uint16_t cpus, cpus_used; | |
| int err_code = SLURM_SUCCESS; | |
| node_ptr = node_record_table_ptr[i]; | |
| gres_test_args.err_code = &err_code; | |
| if (!bit_test(step_ptr->step_node_bitmap, i)) | |
| continue; | |
| if (step_ptr->start_protocol_ver > node_ptr->protocol_version) | |
| step_ptr->start_protocol_ver = | |
| node_ptr->protocol_version; | |
| if (pos >= job_resrcs_ptr->nhosts) | |
| fatal("%s: node index bad", __func__); | |
| cpus = job_resrcs_ptr->cpus[pos]; | |
| cpus_used = job_resrcs_ptr->cpus_used[pos]; | |
| /* | |
| * Here we are trying to figure out the number | |
| * of cpus available if we only want to run 1 | |
| * thread per core. | |
| */ | |
| if (_use_one_thread_per_core(step_ptr)) { | |
| uint16_t threads; | |
| threads = node_ptr->config_ptr->threads; | |
| cpus /= threads; | |
| cpus_used /= threads; | |
| cpus_per_task_array[set_nodes] = cpus_per_task; | |
| } else { | |
| /* | |
| * Here we are trying to figure out how many | |
| * CPUs each task really needs. This really | |
| * only becomes an issue if the job requested | |
| * ntasks_per_core|socket=1. We just increase | |
| * the number of cpus_per_task to the thread | |
| * count. Since the system could be | |
| * heterogeneous, we needed to make this an | |
| * array. | |
| */ | |
| uint16_t threads_per_core; | |
| multi_core_data_t *mc_ptr = NULL; | |
| if (job_ptr->details) | |
| mc_ptr = job_ptr->details->mc_ptr; | |
| if (step_ptr->threads_per_core != NO_VAL16) | |
| threads_per_core = step_ptr->threads_per_core; | |
| else if (mc_ptr && | |
| (mc_ptr->threads_per_core != NO_VAL16)) | |
| threads_per_core = mc_ptr->threads_per_core; | |
| else | |
| threads_per_core = | |
| node_ptr->config_ptr->threads; | |
| cpus_per_task_array[set_nodes] = cpus_per_task; | |
| if ((ntasks_per_core == 1) && | |
| (cpus_per_task < threads_per_core)) | |
| cpus_per_task_array[set_nodes] = | |
| threads_per_core; | |
| } | |
| if (step_ptr->flags & SSF_OVERLAP_FORCE) | |
| usable_cpus = cpus; | |
| else | |
| usable_cpus = cpus - cpus_used; | |
| if ((step_ptr->pn_min_memory & MEM_PER_CPU) && _is_mem_resv()) { | |
| uint64_t mem_use = step_ptr->pn_min_memory; | |
| mem_use &= (~MEM_PER_CPU); | |
| if (job_resrcs_ptr->memory_allocated[pos] > | |
| job_resrcs_ptr->memory_used[pos]) | |
| usable_mem = | |
| job_resrcs_ptr->memory_allocated[pos] - | |
| job_resrcs_ptr->memory_used[pos]; | |
| else | |
| usable_mem = 0; | |
| usable_mem /= mem_use; | |
| usable_cpus = MIN(usable_cpus, usable_mem); | |
| } | |
| gres_test_args.node_offset = pos; | |
| gres_cpus = gres_stepmgr_step_test(&gres_test_args); | |
| if (usable_cpus > 0) | |
| usable_cpus = MIN(usable_cpus, gres_cpus); | |
| if (usable_cpus <= 0) { | |
| error("%s: no usable CPUs", __func__); | |
| if (step_hl) | |
| hostlist_destroy(step_hl); | |
| xfree(step_nodes); | |
| xfree(arbitrary_order); | |
| return NULL; | |
| } | |
| log_flag(STEPS, "%s: %pS node %s job_pos %d usable_cpus %d node_rank %u", | |
| __func__, step_ptr, node_ptr->name, pos, usable_cpus, | |
| job_resrcs_ptr->node_ranks ? | |
| job_resrcs_ptr->node_ranks[pos] : NO_VAL); | |
| cpus_per_node[set_nodes] = usable_cpus; | |
| if (step_hl) | |
| hostlist_push_host(step_hl, node_ptr->name); | |
| set_nodes++; | |
| gres_test_args.first_step_node = false; | |
| gres_test_args.max_rem_nodes--; | |
| #if 0 | |
| /* | |
| * FIXME: on a heterogeneous system running the | |
| * select/linear plugin we could get a node that doesn't | |
| * have as many CPUs as we decided we needed for each | |
| * task. This would result in not getting a task for | |
| * the node we selected. This is usually in error. This | |
| * only happens when the person doesn't specify how many | |
| * cpus_per_task they want, and we have to come up with | |
| * a number, in this case it is wrong. | |
| */ | |
| if (cpus_per_task > 0) { | |
| set_tasks += | |
| (uint16_t)usable_cpus / cpus_per_task; | |
| } else { | |
| /* | |
| * Since cpus_per_task is 0, we just add the | |
| * count of CPUs available for this job | |
| */ | |
| set_tasks += usable_cpus; | |
| } | |
| info("usable_cpus is %d and set_tasks %d %d", | |
| usable_cpus, set_tasks, cpus_per_task); | |
| #endif | |
| if (set_nodes == node_count) | |
| break; | |
| } | |
| /* if (set_tasks < num_tasks) { */ | |
| /* error("Resources only available for %u of %u tasks", */ | |
| /* set_tasks, num_tasks); */ | |
| /* return NULL; */ | |
| /* } */ | |
| /* | |
| * The node list must match the order of the per-node arrays built | |
| * above. Non-arbitrary uses the rank-ordered list gathered above; | |
| * arbitrary uses step_nodes (already built) and passes the raw user | |
| * list separately so task ids follow it. | |
| */ | |
| if (step_hl) { | |
| step_nodes = hostlist_ranged_string_xmalloc(step_hl); | |
| hostlist_destroy(step_hl); | |
| } else { | |
| step_layout_req.arbitrary_nodes = arbitrary_nodes; | |
| } | |
| /* layout the tasks on the nodes */ | |
| step_layout_req.node_list = step_nodes; | |
| step_layout_req.cpus_per_node = cpus_per_node; | |
| step_layout_req.cpus_per_task = cpus_per_task_array; | |
| step_layout_req.num_hosts = node_count; | |
| step_layout_req.num_tasks = num_tasks; | |
| step_layout_req.task_dist = task_dist; | |
| step_layout_req.plane_size = plane_size; | |
| if ((step_layout = slurm_step_layout_create(&step_layout_req))) { | |
| step_layout->start_protocol_ver = step_ptr->start_protocol_ver; | |
| if (job_ptr->node_addrs) | |
| step_layout->alias_addrs = build_alias_addrs(job_ptr); | |
| } | |
| xfree(step_nodes); | |
| xfree(arbitrary_order); | |
| return step_layout; | |
| } | |
| typedef struct { | |
| list_t *dealloc_steps; | |
| node_record_t *node_ptr; | |
| bool node_fail; | |
| } kill_step_on_node_args_t; | |
| static int _kill_step_on_node(void *x, void *arg) | |
| { | |
| step_record_t *step_ptr = (step_record_t *) x; | |
| kill_step_on_node_args_t *args = (kill_step_on_node_args_t *) arg; | |
| int step_node_inx = 0; | |
| int bit_position = args->node_ptr->index; | |
| int rem = 0; | |
| uint32_t step_rc = 0; | |
| step_complete_msg_t req; | |
| if (!IS_STEP_RUNNING(step_ptr)) | |
| return 0; | |
| if (!bit_test(step_ptr->step_node_bitmap, bit_position)) | |
| return 0; | |
| /* | |
| * fake_slurm_step_layout_create() returns NULL on a bad request, and | |
| * the extern, interactive and ext-launcher steps go to JOB_RUNNING | |
| * without checking it. | |
| */ | |
| if (!step_ptr->step_layout) | |
| return 0; | |
| /* | |
| * Don't let the batch step go through partial completion and get marked | |
| * for deallocation while the job is still running. The batch step will | |
| * be cleaned up when the job completes or is requeued. | |
| */ | |
| if (step_ptr->step_id.step_id == SLURM_BATCH_SCRIPT) | |
| return 0; | |
| /* | |
| * Remove step allocation from the job's allocation. range_first/last | |
| * index exit_node_bitmap, which follows the step layout node list | |
| * order, so use the failed node's position in that list. | |
| */ | |
| step_node_inx = nodelist_find(step_ptr->step_layout->node_list, | |
| args->node_ptr->name); | |
| xassert(step_node_inx >= 0); | |
| memset(&req, 0, sizeof(step_complete_msg_t)); | |
| memcpy(&req.step_id, &step_ptr->step_id, sizeof(req.step_id)); | |
| req.range_first = step_node_inx; | |
| req.range_last = step_node_inx; | |
| req.step_rc = 9; | |
| req.jobacct = NULL; /* No accounting */ | |
| (void) _step_partial_comp(step_ptr, &req, false, &rem, &step_rc); | |
| /* | |
| * Do not kill the extern step on all nodes, only on the nodes that | |
| * failed. Otherwise things that rely on the extern step such as x11 | |
| * or namespace plugins won't work on the remaining nodes in the | |
| * allocation. | |
| */ | |
| if (args->node_fail && !(step_ptr->flags & SSF_NO_KILL) && | |
| (step_ptr->step_id.step_id != SLURM_EXTERN_CONT)) { | |
| info("Killing %pS due to failed node %s", | |
| step_ptr, args->node_ptr->name); | |
| signal_step_tasks(step_ptr, SIGKILL, REQUEST_TERMINATE_TASKS); | |
| } else { | |
| info("Killing %pS on failed node %s", | |
| step_ptr, args->node_ptr->name); | |
| signal_step_tasks_on_node(args->node_ptr->name, step_ptr, | |
| SIGKILL, REQUEST_TERMINATE_TASKS); | |
| } | |
| if (!rem) { | |
| if (!args->dealloc_steps) | |
| args->dealloc_steps = list_create(NULL); | |
| list_append(args->dealloc_steps, step_ptr); | |
| } | |
| return 0; | |
| } | |
| /* | |
| * kill_step_on_node - determine if the specified job has any job steps | |
| * allocated to the specified node and kill them unless no_kill flag | |
| * is set on the step | |
| * IN job_ptr - pointer to an active job record | |
| * IN node_ptr - pointer to a node record | |
| * IN node_fail - true of removed node has failed | |
| */ | |
| extern void kill_step_on_node(job_record_t *job_ptr, node_record_t *node_ptr, | |
| bool node_fail) | |
| { | |
| kill_step_on_node_args_t args = { | |
| .dealloc_steps = NULL, | |
| .node_ptr = node_ptr, | |
| .node_fail = node_fail, | |
| }; | |
| if (!job_ptr || !node_ptr) | |
| return; | |
| list_for_each(job_ptr->step_list, _kill_step_on_node, &args); | |
| if (args.dealloc_steps) { | |
| /* | |
| * Because _finish_step_comp() may free the step_ptr, call | |
| * list_delete_all() to delete the list-node when the step_ptr | |
| * is free'd. It doesn't actually matter because we are | |
| * deleting the list immediately afterward, but it is good | |
| * practice to not leave invalid pointer references. | |
| */ | |
| list_delete_all(args.dealloc_steps, _finish_step_comp, NULL); | |
| FREE_NULL_LIST(args.dealloc_steps); | |
| } | |
| } | |
| extern int stepmgr_kill_steps_on_resize(job_record_t *job_ptr, char *node_list) | |
| { | |
| hostlist_t *hl; | |
| char *node_name; | |
| job_resources_t *job_resrcs = job_ptr->job_resrcs; | |
| bitstr_t *orig_job_node_bitmap = bit_copy(job_resrcs->node_bitmap); | |
| log_flag(STEPS, "%s: %pJ removed_nodes=%s", | |
| __func__, job_ptr, node_list); | |
| hl = hostlist_create(node_list); | |
| while ((node_name = hostlist_shift(hl))) { | |
| node_record_t *node_ptr = find_node_record(node_name); | |
| if (node_ptr) { | |
| int job_node_inx; | |
| kill_step_on_node(job_ptr, node_ptr, false); | |
| /* | |
| * Excise the removed node from the stepmgr's local | |
| * view of the job so subsequent step launches against | |
| * this job are allowed only in surviving nodes. The | |
| * job resources are excised after the steps, which | |
| * still need the node's cpus and memory to be | |
| * deallocated. | |
| */ | |
| job_node_inx = | |
| job_get_node_inx(node_ptr->name, | |
| job_resrcs->node_bitmap); | |
| if (job_node_inx >= 0) { | |
| gres_stepmgr_job_dealloc( | |
| job_ptr->gres_list_alloc, | |
| node_ptr->gres_list, job_node_inx, | |
| job_ptr->job_id, node_ptr->name, false, | |
| true); | |
| extract_job_resources_node(job_resrcs, | |
| job_node_inx); | |
| } | |
| if (job_ptr->node_bitmap && | |
| bit_test(job_ptr->node_bitmap, node_ptr->index)) { | |
| bit_clear(job_ptr->node_bitmap, | |
| node_ptr->index); | |
| if (job_ptr->node_cnt) | |
| job_ptr->node_cnt--; | |
| } | |
| } | |
| xfree(node_name); | |
| } | |
| hostlist_destroy(hl); | |
| xfree(job_ptr->nodes); | |
| job_ptr->nodes = bitmap2node_name_sortable(job_ptr->node_bitmap, false); | |
| rebuild_step_bitmaps(job_ptr, orig_job_node_bitmap); | |
| FREE_NULL_BITMAP(orig_job_node_bitmap); | |
| return SLURM_SUCCESS; | |
| } | |
| extern int step_partial_comp(slurm_msg_t *msg, step_complete_msg_t *req, | |
| bool finish, int *rem, uint32_t *max_rc) | |
| { | |
| job_record_t *job_ptr; | |
| step_record_t *step_ptr; | |
| xassert(rem); | |
| /* find the job, step, and validate input */ | |
| if (!(job_ptr = stepmgr_ops->find_job(&req->step_id))) { | |
| info("%s: %pI invalid", __func__, &req->step_id); | |
| return ESLURM_INVALID_JOB_ID; | |
| } | |
| /* If we are requeuing the job the completing flag will be set | |
| * but the state will be Pending, so don't use IS_JOB_PENDING | |
| * which won't see the completing flag. | |
| */ | |
| if (job_ptr->job_state == JOB_PENDING) { | |
| info("%s: %pJ pending", __func__, job_ptr); | |
| return ESLURM_JOB_PENDING; | |
| } | |
| if (!validate_job_owner(msg, job_ptr)) { | |
| /* Normally from slurmstepd, from srun on some failures */ | |
| error("Security violation: REQUEST_STEP_COMPLETE RPC for %pJ from uid=%u", | |
| job_ptr, msg->auth.uid); | |
| return ESLURM_USER_ID_MISSING; | |
| } | |
| step_ptr = find_step_record(job_ptr, &req->step_id); | |
| if (step_ptr == NULL) { | |
| info("step_partial_comp: %pJ StepID=%u invalid; this step may have already completed", | |
| job_ptr, req->step_id.step_id); | |
| return ESLURM_INVALID_JOB_ID; | |
| } | |
| if (req->range_last < req->range_first) { | |
| error("%s: %pS range=%u-%u", | |
| __func__, step_ptr, req->range_first, req->range_last); | |
| return EINVAL; | |
| } | |
| return _step_partial_comp(step_ptr, req, finish, rem, max_rc); | |
| } | |
| static int _step_partial_comp(step_record_t *step_ptr, | |
| step_complete_msg_t *req, bool finish, | |
| int *rem, uint32_t *max_rc) | |
| { | |
| int nodes, rem_nodes; | |
| int range_bits, set_bits; | |
| if (step_ptr->step_id.step_id == SLURM_BATCH_SCRIPT) { | |
| error("%s: batch step received for %pJ. This should never happen.", | |
| __func__, step_ptr->job_ptr); | |
| return ESLURM_INVALID_JOB_ID; | |
| } | |
| /* we have been adding task average frequencies for | |
| * jobacct->act_cpufreq so we need to divide with the | |
| * total number of tasks/cpus for the step average frequency */ | |
| if (step_ptr->cpu_count && step_ptr->jobacct) | |
| step_ptr->jobacct->act_cpufreq /= step_ptr->cpu_count; | |
| if (!step_ptr->exit_node_bitmap) { | |
| /* initialize the node bitmap for exited nodes */ | |
| nodes = bit_set_count(step_ptr->step_node_bitmap); | |
| step_ptr->exit_node_bitmap = bit_alloc(nodes); | |
| step_ptr->exit_code = req->step_rc; | |
| } else { | |
| nodes = bit_size(step_ptr->exit_node_bitmap); | |
| if ((req->step_rc == SIG_OOM) || | |
| (req->step_rc > step_ptr->exit_code)) | |
| step_ptr->exit_code = req->step_rc; | |
| } | |
| if ((req->range_first >= nodes) || (req->range_last >= nodes) || | |
| (req->range_first > req->range_last)) { | |
| /* range is zero origin */ | |
| error("%s: %pS range=%u-%u nodes=%d", | |
| __func__, step_ptr, req->range_first, req->range_last, | |
| nodes); | |
| return EINVAL; | |
| } | |
| if ((step_ptr->flags & SSF_NO_SIG_FAIL) && WIFSIGNALED(req->step_rc)) { | |
| step_ptr->exit_code = 0; | |
| } | |
| range_bits = req->range_last + 1 - req->range_first; | |
| set_bits = bit_set_count_range(step_ptr->exit_node_bitmap, | |
| req->range_first, | |
| req->range_last + 1); | |
| /* Check if any stepd of the range was already received */ | |
| if (set_bits) { | |
| /* If all are already received skip jobacctinfo_aggregate */ | |
| if (set_bits == range_bits) { | |
| debug("Step complete from %d to %d was already processed. Probably a RPC was resent from a child.", | |
| req->range_first, req->range_last); | |
| goto no_aggregate; | |
| } | |
| /* | |
| * If partially received, we cannot recover the right gathered | |
| * information. If we don't gather the new one we'll miss some | |
| * information, and if we gather it some of the info will be | |
| * duplicated. We log that error and chose to partially | |
| * duplicate because it's probably a smaller error. | |
| */ | |
| error("Step complete from %d to %d was already processed (%d of %d). Probably a RPC was resent from a child and gathered information is partially duplicated.", | |
| req->range_first, req->range_last, | |
| set_bits, range_bits); | |
| } | |
| bit_nset(step_ptr->exit_node_bitmap, | |
| req->range_first, req->range_last); | |
| if (step_ptr->jobacct && req->jobacct) | |
| jobacctinfo_aggregate(step_ptr->jobacct, req->jobacct); | |
| no_aggregate: | |
| rem_nodes = bit_clear_count(step_ptr->exit_node_bitmap); | |
| *rem = rem_nodes; | |
| if (rem_nodes == 0) { | |
| /* release all switch windows */ | |
| if (step_ptr->switch_step) { | |
| debug2("full switch release for %pS, nodes %s", | |
| step_ptr, step_ptr->step_layout->node_list); | |
| switch_g_job_step_complete( | |
| step_ptr->switch_step, | |
| step_ptr->step_layout->node_list); | |
| switch_g_stepinfo_free(step_ptr->switch_step); | |
| step_ptr->switch_step = NULL; | |
| } | |
| } | |
| if (max_rc) | |
| *max_rc = step_ptr->exit_code; | |
| if (req->step_rc == ESLURMD_EXECVE_FAILED) | |
| step_ptr->state = JOB_NODE_FAIL; | |
| /* The step has finished, finish it completely */ | |
| if (!*rem && finish) { | |
| (void) _finish_step_comp(step_ptr, NULL); | |
| } | |
| return SLURM_SUCCESS; | |
| } | |
| /* | |
| * step_set_alloc_tres - set the tres up when allocating the step. | |
| * Only set when job is running. | |
| */ | |
| extern void step_set_alloc_tres(step_record_t *step_ptr, uint32_t node_count, | |
| bool assoc_mgr_locked, bool make_formatted) | |
| { | |
| uint64_t cpu_count = 1, mem_count = 0; | |
| char *tmp_tres_str = NULL; | |
| assoc_mgr_lock_t locks = { .tres = READ_LOCK }; | |
| job_record_t *job_ptr = step_ptr->job_ptr; | |
| xassert(step_ptr); | |
| xfree(step_ptr->tres_alloc_str); | |
| xfree(step_ptr->tres_fmt_alloc_str); | |
| if (((step_ptr->step_id.step_id == SLURM_EXTERN_CONT) || | |
| (step_ptr->flags & SSF_EXT_LAUNCHER)) && | |
| job_ptr->tres_alloc_str) { | |
| /* get the tres from the whole job */ | |
| step_ptr->tres_alloc_str = | |
| xstrdup(job_ptr->tres_alloc_str); | |
| if (make_formatted) | |
| step_ptr->tres_fmt_alloc_str = | |
| xstrdup(job_ptr->tres_fmt_alloc_str); | |
| return; | |
| } | |
| if (!assoc_mgr_locked) | |
| assoc_mgr_lock(&locks); | |
| if (((step_ptr->step_id.step_id == SLURM_BATCH_SCRIPT) || | |
| (step_ptr->step_id.step_id == SLURM_INTERACTIVE_STEP)) && | |
| job_ptr->job_resrcs) { | |
| int batch_inx = 0; | |
| /* | |
| * Figure out the index for the batch_host in relation to the | |
| * job specific job_resrcs structure. | |
| */ | |
| if (job_ptr->batch_host) { | |
| batch_inx = job_get_node_inx( | |
| job_ptr->batch_host, job_ptr->node_bitmap); | |
| if (batch_inx == -1) { | |
| error("%s: Invalid batch host %s for %pJ; this should never happen", | |
| __func__, job_ptr->batch_host, job_ptr); | |
| batch_inx = 0; | |
| } | |
| } | |
| /* get the cpus and memory on the first node */ | |
| if (job_ptr->job_resrcs->cpus) | |
| cpu_count = job_ptr->job_resrcs->cpus[batch_inx]; | |
| if (job_ptr->job_resrcs->memory_allocated) | |
| mem_count = job_ptr->job_resrcs-> | |
| memory_allocated[batch_inx]; | |
| tmp_tres_str = gres_stepmgr_gres_on_node_as_tres( | |
| job_ptr->gres_list_alloc, 0, true); | |
| } else { | |
| if (!step_ptr->step_layout || !step_ptr->step_layout->task_cnt) | |
| cpu_count = (uint64_t)job_ptr->total_cpus; | |
| else | |
| cpu_count = (uint64_t)step_ptr->cpu_count; | |
| for (int i = 0; i < bit_set_count(step_ptr->step_node_bitmap); | |
| i++) | |
| mem_count += step_ptr->memory_allocated[i]; | |
| tmp_tres_str = gres_stepmgr_gres_2_tres_str( | |
| step_ptr->gres_list_alloc, true); | |
| } | |
| xstrfmtcat(step_ptr->tres_alloc_str, | |
| "%s%u=%"PRIu64",%u=%"PRIu64",%u=%u", | |
| step_ptr->tres_alloc_str ? "," : "", | |
| TRES_CPU, cpu_count, | |
| TRES_MEM, mem_count, | |
| TRES_NODE, node_count); | |
| if (tmp_tres_str) { | |
| xstrfmtcat(step_ptr->tres_alloc_str, "%s%s", | |
| step_ptr->tres_alloc_str ? "," : "", | |
| tmp_tres_str); | |
| xfree(tmp_tres_str); | |
| } | |
| if (make_formatted) | |
| step_ptr->tres_fmt_alloc_str = | |
| slurmdb_make_tres_string_from_simple( | |
| step_ptr->tres_alloc_str, assoc_mgr_tres_list, | |
| NO_VAL, CONVERT_NUM_UNIT_EXACT, 0, NULL); | |
| if (!assoc_mgr_locked) | |
| assoc_mgr_unlock(&locks); | |
| } | |
| static int _suspend_job_step(void *x, void *arg) | |
| { | |
| step_record_t *step_ptr = (step_record_t *) x; | |
| job_record_t *job_ptr = step_ptr->job_ptr; | |
| time_t *now = (time_t *) arg; | |
| if (!IS_STEP_RUNNING(step_ptr)) | |
| return 0; | |
| if ((job_ptr->suspend_time) && | |
| (job_ptr->suspend_time > step_ptr->start_time)) { | |
| step_ptr->pre_sus_time += | |
| difftime(*now, job_ptr->suspend_time); | |
| } else { | |
| step_ptr->pre_sus_time += | |
| difftime(*now, step_ptr->start_time); | |
| } | |
| return 0; | |
| } | |
| /* Update time stamps for job step suspend */ | |
| extern void suspend_job_step(job_record_t *job_ptr) | |
| { | |
| time_t now = time(NULL); | |
| list_for_each(job_ptr->step_list, _suspend_job_step, &now); | |
| } | |
| static int _resume_job_step(void *x, void *arg) | |
| { | |
| step_record_t *step_ptr = (step_record_t *) x; | |
| job_record_t *job_ptr = (job_record_t *) step_ptr->job_ptr; | |
| time_t *now = (time_t *) arg; | |
| if (!IS_STEP_RUNNING(step_ptr)) | |
| return 0; | |
| if ((job_ptr->suspend_time) && | |
| (job_ptr->suspend_time < step_ptr->start_time)) { | |
| step_ptr->tot_sus_time += | |
| difftime(*now, step_ptr->start_time); | |
| } else { | |
| step_ptr->tot_sus_time += | |
| difftime(*now, job_ptr->suspend_time); | |
| } | |
| return 0; | |
| } | |
| /* Update time stamps for job step resume */ | |
| extern void resume_job_step(job_record_t *job_ptr) | |
| { | |
| time_t now = time(NULL); | |
| list_for_each(job_ptr->step_list, _resume_job_step, &now); | |
| } | |
| static void _signal_step_timelimit(step_record_t *step_ptr, time_t now) | |
| { | |
| node_record_t *node_ptr; | |
| static bool cloud_dns = false; | |
| static time_t last_update = 0; | |
| job_record_t *job_ptr = step_ptr->job_ptr; | |
| kill_job_msg_t *kill_step; | |
| agent_arg_t *agent_args = NULL; | |
| step_ptr->state = JOB_TIMEOUT; | |
| xassert(step_ptr); | |
| agent_args = xmalloc(sizeof(agent_arg_t)); | |
| agent_args->msg_type = REQUEST_KILL_TIMELIMIT; | |
| agent_args->retry = 1; | |
| agent_args->hostlist = hostlist_create(NULL); | |
| kill_step = xmalloc(sizeof(kill_job_msg_t)); | |
| memcpy(&kill_step->step_id, &step_ptr->step_id, | |
| sizeof(kill_step->step_id)); | |
| kill_step->het_job_id = job_ptr->het_job_id; | |
| kill_step->job_state = job_ptr->job_state; | |
| kill_step->job_uid = job_ptr->user_id; | |
| kill_step->job_gid = job_ptr->group_id; | |
| kill_step->nodes = xstrdup(job_ptr->nodes); | |
| kill_step->time = now; | |
| kill_step->start_time = job_ptr->start_time; | |
| kill_step->details = xstrdup(job_ptr->state_desc); | |
| if (last_update != slurm_conf.last_update) { | |
| if (xstrcasestr(slurm_conf.slurmctld_params, "cloud_dns")) | |
| cloud_dns = true; | |
| else | |
| cloud_dns = false; | |
| last_update = slurm_conf.last_update; | |
| } | |
| if (step_ptr->step_node_bitmap) { | |
| agent_args->protocol_version = SLURM_PROTOCOL_VERSION; | |
| for (int i = 0; | |
| (node_ptr = next_node_bitmap(step_ptr->step_node_bitmap, | |
| &i)); | |
| i++) { | |
| if (agent_args->protocol_version > | |
| node_ptr->protocol_version) { | |
| agent_args->protocol_version = | |
| node_ptr->protocol_version; | |
| } | |
| hostlist_push_host(agent_args->hostlist, | |
| node_ptr->name); | |
| agent_args->node_count++; | |
| if (PACK_FANOUT_ADDRS(node_ptr)) | |
| agent_args->msg_flags |= SLURM_PACK_ADDRS; | |
| } | |
| } else { | |
| /* Could happen on node failure */ | |
| info("%s: %pJ Step %u has NULL node_bitmap", __func__, | |
| job_ptr, step_ptr->step_id.step_id); | |
| } | |
| if (agent_args->node_count == 0) { | |
| hostlist_destroy(agent_args->hostlist); | |
| xfree(agent_args); | |
| slurm_free_kill_job_msg(kill_step); | |
| return; | |
| } | |
| _set_resolve_hostlist(job_ptr, agent_args); | |
| agent_args->msg_args = kill_step; | |
| set_agent_arg_r_uid(agent_args, SLURM_AUTH_UID_ANY); | |
| stepmgr_ops->agent_queue_request(agent_args); | |
| } | |
| extern int check_job_step_time_limit(void *x, void *arg) | |
| { | |
| step_record_t *step_ptr = (step_record_t *) x; | |
| time_t *now = (time_t *) arg; | |
| uint32_t job_run_mins = 0; | |
| if (!IS_STEP_RUNNING(step_ptr)) | |
| return 0; | |
| if (step_ptr->time_limit == INFINITE || step_ptr->time_limit == NO_VAL) | |
| return 0; | |
| job_run_mins = (uint32_t) (((*now - step_ptr->start_time) - | |
| step_ptr->tot_sus_time) / 60); | |
| if (job_run_mins >= step_ptr->time_limit) { | |
| /* this step has timed out */ | |
| info("%s: %pS has timed out (%u)", | |
| __func__, step_ptr, step_ptr->time_limit); | |
| _signal_step_timelimit(step_ptr, *now); | |
| } | |
| return 0; | |
| } | |
| /* Return true if memory is a reserved resources, false otherwise */ | |
| static bool _is_mem_resv(void) | |
| { | |
| static bool mem_resv_value = false; | |
| static bool mem_resv_tested = false; | |
| if (!mem_resv_tested) { | |
| mem_resv_tested = true; | |
| if (slurm_conf.select_type_param & SELECT_MEMORY) | |
| mem_resv_value = true; | |
| } | |
| return mem_resv_value; | |
| } | |
| typedef struct { | |
| int mod_cnt; | |
| uint32_t time_limit; | |
| } update_step_args_t; | |
| static int _update_step(void *x, void *arg) | |
| { | |
| step_record_t *step_ptr = (step_record_t *) x; | |
| update_step_args_t *args = (update_step_args_t *) arg; | |
| if (!IS_STEP_RUNNING(step_ptr)) | |
| return 0; | |
| step_ptr->time_limit = args->time_limit; | |
| args->mod_cnt++; | |
| jobacct_storage_g_step_start(stepmgr_ops->acct_db_conn, step_ptr); | |
| info("Updating %pS time limit to %u", step_ptr, args->time_limit); | |
| return 0; | |
| } | |
| /* | |
| * Process job step update request from specified user, | |
| * RET - 0 or error code | |
| */ | |
| extern int update_step(step_update_request_msg_t *req) | |
| { | |
| job_record_t *job_ptr; | |
| step_record_t *step_ptr = NULL; | |
| update_step_args_t args = { .mod_cnt = 0 }; | |
| slurm_step_id_t step_id = SLURM_STEP_ID_INITIALIZER; | |
| if (!(job_ptr = stepmgr_ops->find_job(&req->step_id))) { | |
| error("%s: invalid %pI", __func__, &req->step_id); | |
| return ESLURM_INVALID_JOB_ID; | |
| } | |
| step_id = STEP_ID_FROM_JOB_RECORD(job_ptr); | |
| step_id.step_id = req->step_id.step_id; | |
| /* | |
| * No need to limit step time limit as job time limit will kill | |
| * any steps with any time limit | |
| */ | |
| if (req->step_id.step_id == NO_VAL) { | |
| args.time_limit = req->time_limit; | |
| list_for_each(job_ptr->step_list, _update_step, &args); | |
| } else { | |
| step_ptr = find_step_record(job_ptr, &step_id); | |
| if (!step_ptr && (job_ptr->bit_flags & STEPMGR_ENABLED)) | |
| goto stepmgr; | |
| if (!step_ptr) | |
| return ESLURM_INVALID_JOB_ID; | |
| /* A queued step is now addressable but has nothing to start. */ | |
| if (req->time_limit && IS_STEP_RUNNING(step_ptr)) { | |
| step_ptr->time_limit = req->time_limit; | |
| args.mod_cnt++; | |
| jobacct_storage_g_step_start(stepmgr_ops->acct_db_conn, | |
| step_ptr); | |
| info("Updating %pS time limit to %u", | |
| step_ptr, req->time_limit); | |
| } | |
| } | |
| stepmgr: | |
| if (running_in_slurmctld() && !step_ptr && | |
| (job_ptr->bit_flags & STEPMGR_ENABLED)) { | |
| agent_arg_t *agent_args = NULL; | |
| step_update_request_msg_t *agent_update_msg = NULL; | |
| agent_update_msg = xmalloc(sizeof(*agent_update_msg)); | |
| agent_update_msg->step_id = req->step_id; | |
| agent_update_msg->time_limit = req->time_limit; | |
| agent_args = xmalloc(sizeof(*agent_args)); | |
| agent_args->msg_type = REQUEST_UPDATE_JOB_STEP; | |
| agent_args->retry = 1; | |
| agent_args->hostlist = hostlist_create(job_ptr->batch_host); | |
| agent_args->node_count = 1; | |
| agent_args->protocol_version = SLURM_PROTOCOL_VERSION; | |
| agent_args->msg_args = agent_update_msg; | |
| set_agent_arg_r_uid(agent_args, slurm_conf.slurmd_user_id); | |
| stepmgr_ops->agent_queue_request(agent_args); | |
| args.mod_cnt++; | |
| } | |
| if (args.mod_cnt) | |
| *stepmgr_ops->last_job_update = time(NULL); | |
| return SLURM_SUCCESS; | |
| } | |
| static int _rebuild_bitmaps(void *x, void *arg) | |
| { | |
| int i_first, i_last, i_size; | |
| int old_core_offset = 0, new_core_offset = 0; | |
| bool old_node_set, new_node_set; | |
| uint32_t step_id; | |
| bitstr_t *orig_step_core_bitmap; | |
| step_record_t *step_ptr = (step_record_t *) x; | |
| bitstr_t *orig_job_node_bitmap = (bitstr_t *) arg; | |
| job_record_t *job_ptr = step_ptr->job_ptr; | |
| if (step_ptr->state < JOB_RUNNING) | |
| return 0; | |
| gres_stepmgr_step_state_rebase(step_ptr->gres_list_alloc, | |
| orig_job_node_bitmap, | |
| job_ptr->job_resrcs->node_bitmap); | |
| if (!step_ptr->core_bitmap_job) | |
| return 0; | |
| step_id = step_ptr->step_id.step_id; | |
| orig_step_core_bitmap = step_ptr->core_bitmap_job; | |
| i_size = bit_size(job_ptr->job_resrcs->core_bitmap); | |
| step_ptr->core_bitmap_job = bit_alloc(i_size); | |
| i_first = MIN(bit_ffs(orig_job_node_bitmap), | |
| bit_ffs(job_ptr->job_resrcs->node_bitmap)); | |
| i_last = MAX(bit_fls(orig_job_node_bitmap), | |
| bit_fls(job_ptr->job_resrcs->node_bitmap)); | |
| for (int i = i_first; i <= i_last; i++) { | |
| old_node_set = bit_test(orig_job_node_bitmap, i); | |
| new_node_set = bit_test(job_ptr->job_resrcs->node_bitmap, i); | |
| if (!old_node_set && !new_node_set) | |
| continue; | |
| if (old_node_set && new_node_set) { | |
| for (int j = 0; j < node_record_table_ptr[i]->tot_cores; | |
| j++) { | |
| if (!bit_test(orig_step_core_bitmap, | |
| old_core_offset + j)) | |
| continue; | |
| bit_set(step_ptr->core_bitmap_job, | |
| new_core_offset + j); | |
| /* | |
| * Only regular, non-overlapping steps should | |
| * set bits in core_bitmap_used | |
| */ | |
| if ((step_id != SLURM_INTERACTIVE_STEP) && | |
| (step_id != SLURM_EXTERN_CONT) && | |
| (step_id != SLURM_BATCH_SCRIPT) && | |
| !(step_ptr->flags & SSF_OVERLAP_FORCE) && | |
| !(step_ptr->flags & SSF_EXT_LAUNCHER)) | |
| bit_set(job_ptr->job_resrcs-> | |
| core_bitmap_used, | |
| new_core_offset + j); | |
| } | |
| } | |
| if (old_node_set) | |
| old_core_offset += node_record_table_ptr[i]->tot_cores; | |
| if (new_node_set) | |
| new_core_offset += node_record_table_ptr[i]->tot_cores; | |
| } | |
| FREE_NULL_BITMAP(orig_step_core_bitmap); | |
| return 0; | |
| } | |
| /* | |
| * Rebuild a job step's core_bitmap_job after a job has just changed size | |
| * job_ptr IN - job that was just re-sized | |
| * orig_job_node_bitmap IN - The job's original node bitmap | |
| */ | |
| extern void rebuild_step_bitmaps(job_record_t *job_ptr, | |
| bitstr_t *orig_job_node_bitmap) | |
| { | |
| if (job_ptr->step_list == NULL) | |
| return; | |
| log_flag(STEPS, "Resizing steps of %pJ", job_ptr); | |
| list_for_each(job_ptr->step_list, _rebuild_bitmaps, | |
| orig_job_node_bitmap); | |
| } | |
| extern step_record_t *build_extern_step(job_record_t *job_ptr) | |
| { | |
| step_record_t *step_ptr = create_step_record(job_ptr, 0); | |
| char *node_list; | |
| uint32_t node_cnt; | |
| node_list = job_ptr->nodes; | |
| node_cnt = job_ptr->node_cnt; | |
| if (!step_ptr) { | |
| error("%s: Can't create step_record! This should never happen", | |
| __func__); | |
| return NULL; | |
| } | |
| *stepmgr_ops->last_job_update = time(NULL); | |
| step_ptr->step_layout = fake_slurm_step_layout_create( | |
| node_list, NULL, NULL, node_cnt, node_cnt, | |
| SLURM_PROTOCOL_VERSION); | |
| step_ptr->name = xstrdup("extern"); | |
| step_ptr->max_npids = job_ptr->details->max_npids; | |
| step_ptr->state = JOB_RUNNING; | |
| step_ptr->start_time = job_ptr->start_time; | |
| step_ptr->step_id = STEP_ID_FROM_JOB_RECORD(job_ptr); | |
| step_ptr->step_id.step_id = SLURM_EXTERN_CONT; | |
| if (job_ptr->node_bitmap) | |
| step_ptr->step_node_bitmap = | |
| bit_copy(job_ptr->node_bitmap); | |
| step_ptr->time_last_active = time(NULL); | |
| step_set_alloc_tres(step_ptr, 1, false, false); | |
| jobacct_storage_g_step_start(stepmgr_ops->acct_db_conn, step_ptr); | |
| return step_ptr; | |
| } | |
| extern step_record_t *build_batch_step(job_record_t *job_ptr_in) | |
| { | |
| job_record_t *job_ptr; | |
| step_record_t *step_ptr; | |
| slurm_step_id_t step_id = { 0 }; | |
| char *host = NULL; | |
| if (job_ptr_in->het_job_id) { | |
| job_ptr = stepmgr_ops->find_job_record(job_ptr_in->het_job_id); | |
| if (!job_ptr) { | |
| error("%s: hetjob leader is corrupt! This should never happen", | |
| __func__); | |
| job_ptr = job_ptr_in; | |
| } | |
| } else | |
| job_ptr = job_ptr_in; | |
| step_id = STEP_ID_FROM_JOB_RECORD(job_ptr); | |
| step_id.step_id = SLURM_BATCH_SCRIPT; | |
| if ((step_ptr = find_step_record(job_ptr, &step_id))) | |
| return step_ptr; | |
| step_ptr = create_step_record(job_ptr, 0); | |
| if (!step_ptr) { | |
| error("%s: Can't create step_record! This should never happen", | |
| __func__); | |
| return NULL; | |
| } | |
| *stepmgr_ops->last_job_update = time(NULL); | |
| host = job_ptr->batch_host; | |
| step_ptr->step_layout = fake_slurm_step_layout_create( | |
| host, NULL, NULL, 1, 1, SLURM_PROTOCOL_VERSION); | |
| step_ptr->name = xstrdup("batch"); | |
| step_ptr->max_npids = job_ptr->details->max_npids; | |
| step_ptr->state = JOB_RUNNING; | |
| step_ptr->start_time = job_ptr->start_time; | |
| step_ptr->step_id = STEP_ID_FROM_JOB_RECORD(job_ptr); | |
| step_ptr->step_id.step_id = SLURM_BATCH_SCRIPT; | |
| step_ptr->container = xstrdup(job_ptr->container); | |
| step_ptr->container_id = xstrdup(job_ptr->container_id); | |
| step_ptr->runtime = xstrdup(job_ptr->runtime); | |
| if (node_name2bitmap(job_ptr->batch_host, false, | |
| &step_ptr->step_node_bitmap, NULL)) { | |
| error("%s: %pJ has invalid node list (%s)", | |
| __func__, job_ptr, job_ptr->batch_host); | |
| } | |
| step_ptr->time_last_active = time(NULL); | |
| step_set_alloc_tres(step_ptr, 1, false, false); | |
| jobacct_storage_g_step_start(stepmgr_ops->acct_db_conn, step_ptr); | |
| return step_ptr; | |
| } | |
| static step_record_t *_build_interactive_step(job_record_t *job_ptr_in, | |
| step_create_req_msg_t *step_specs, | |
| uint16_t protocol_version) | |
| { | |
| job_record_t *job_ptr; | |
| step_record_t *step_ptr; | |
| char *host = NULL; | |
| slurm_step_id_t step_id = SLURM_STEP_ID_INITIALIZER; | |
| if (job_ptr_in->het_job_id) { | |
| job_ptr = stepmgr_ops->find_job_record(job_ptr_in->het_job_id); | |
| if (!job_ptr) { | |
| error("%s: hetjob leader is corrupt! This should never happen", | |
| __func__); | |
| job_ptr = job_ptr_in; | |
| } | |
| } else | |
| job_ptr = job_ptr_in; | |
| step_id = STEP_ID_FROM_JOB_RECORD(job_ptr); | |
| step_id.step_id = SLURM_INTERACTIVE_STEP; | |
| step_ptr = find_step_record(job_ptr, &step_id); | |
| if (step_ptr) { | |
| debug("%s: interactive step for %pJ already exists", | |
| __func__, job_ptr); | |
| return NULL; | |
| } | |
| host = job_ptr->batch_host; | |
| if (!host) { | |
| error("%s: %pJ batch_host is NULL! This should never happen", | |
| __func__, job_ptr); | |
| return NULL; | |
| } | |
| step_ptr = create_step_record(job_ptr, protocol_version); | |
| if (!step_ptr) { | |
| error("%s: Can't create step_record! This should never happen", | |
| __func__); | |
| return NULL; | |
| } | |
| *stepmgr_ops->last_job_update = time(NULL); | |
| step_ptr->step_layout = fake_slurm_step_layout_create( | |
| host, NULL, NULL, 1, 1, protocol_version); | |
| step_ptr->name = xstrdup("interactive"); | |
| _set_step_max_npids(step_ptr, job_ptr, step_specs); | |
| step_ptr->state = JOB_RUNNING; | |
| step_ptr->start_time = job_ptr->start_time; | |
| step_ptr->step_id = STEP_ID_FROM_JOB_RECORD(job_ptr); | |
| step_ptr->step_id.step_id = SLURM_INTERACTIVE_STEP; | |
| step_ptr->container = xstrdup(job_ptr->container); | |
| step_ptr->container_id = xstrdup(job_ptr->container_id); | |
| step_ptr->runtime = xstrdup(job_ptr->runtime); | |
| step_ptr->port = step_specs->port; | |
| step_ptr->srun_pid = step_specs->srun_pid; | |
| step_ptr->host = xstrdup(step_specs->host); | |
| _set_step_tls_cert(step_ptr, step_specs); | |
| step_ptr->submit_line = xstrdup(step_specs->submit_line); | |
| if (switch_g_setup_special_steps()) { | |
| if (_switch_setup(step_ptr) != SLURM_SUCCESS) | |
| goto fail; | |
| } | |
| step_ptr->core_bitmap_job = bit_copy(job_ptr->job_resrcs->core_bitmap); | |
| if (node_name2bitmap(job_ptr->batch_host, false, | |
| &step_ptr->step_node_bitmap, NULL)) { | |
| error("%s: %pJ has invalid node list (%s)", | |
| __func__, job_ptr, job_ptr->batch_host); | |
| goto fail; | |
| } | |
| step_ptr->time_last_active = time(NULL); | |
| step_set_alloc_tres(step_ptr, 1, false, false); | |
| jobacct_storage_g_step_start(stepmgr_ops->acct_db_conn, step_ptr); | |
| return step_ptr; | |
| fail: | |
| delete_step_record(job_ptr, step_ptr); | |
| return NULL; | |
| } | |
| /* | |
| * Build a special step for mpi launchers. | |
| */ | |
| static int _build_ext_launcher_step(step_record_t **step_rec, | |
| job_record_t *job_ptr, | |
| step_create_req_msg_t *step_specs, | |
| uint16_t protocol_version) | |
| { | |
| bitstr_t *nodeset; | |
| uint32_t node_count; | |
| int rc; | |
| char *step_node_list; | |
| step_record_t *step_ptr; | |
| if (!step_rec) | |
| return SLURM_ERROR; | |
| if (job_ptr->next_step_id >= slurm_conf.max_step_cnt) { | |
| error("%s: %pJ MaxStepCount limit reached", __func__, job_ptr); | |
| return ESLURM_STEP_LIMIT; | |
| } | |
| /* Reset some fields we're going to ignore in _pick_step_nodes. */ | |
| step_specs->flags = SSF_EXT_LAUNCHER; | |
| step_specs->cpu_count = 0; | |
| xfree(step_specs->cpus_per_tres); | |
| step_specs->ntasks_per_core = NO_VAL16; | |
| step_specs->ntasks_per_tres = NO_VAL16; | |
| step_specs->pn_min_memory = 0; | |
| xfree(step_specs->mem_per_tres); | |
| step_specs->threads_per_core = NO_VAL16; | |
| xfree(step_specs->tres_bind); | |
| xfree(step_specs->tres_per_step); | |
| xfree(step_specs->tres_per_node); | |
| xfree(step_specs->tres_per_socket); | |
| xfree(step_specs->tres_per_task); | |
| /* Select the nodes for this job */ | |
| nodeset = _pick_step_nodes(job_ptr, step_specs, NULL, 0, 0, &rc); | |
| if (nodeset == NULL) { | |
| return rc; | |
| } | |
| /* | |
| * Here is where the node list is set for the step. An external | |
| * launcher maps its ranks onto this list, so emit it in topology-rank | |
| * order like any other step. | |
| */ | |
| if (step_specs->node_list && | |
| ((step_specs->task_dist & SLURM_DIST_STATE_BASE) == | |
| SLURM_DIST_ARBITRARY)) { | |
| hostlist_t *hl = hostlist_deduplicate(step_specs->node_list); | |
| /* One task per node, so the user's order is enough. */ | |
| step_node_list = hostlist_ranged_string_xmalloc(hl); | |
| hostlist_destroy(hl); | |
| } else { | |
| step_node_list = | |
| job_resources_node_list_by_rank(job_ptr->job_resrcs, | |
| nodeset); | |
| } | |
| log_flag(STEPS, "%s: %pJ picked ext-launcher nodes %s", | |
| __func__, job_ptr, step_node_list); | |
| step_ptr = *step_rec = create_step_record(job_ptr, protocol_version); | |
| if (!step_ptr) { | |
| error("%s: Can't create step_record! This should never happen", | |
| __func__); | |
| return SLURM_ERROR; | |
| } | |
| *stepmgr_ops->last_job_update = time(NULL); | |
| _set_step_max_npids(step_ptr, job_ptr, step_specs); | |
| /* We want 1 task per node. */ | |
| step_ptr->step_node_bitmap = nodeset; | |
| node_count = bit_set_count(nodeset); | |
| step_specs->num_tasks = node_count; | |
| /* Create the fake step layout with 1 task per node */ | |
| step_ptr->step_layout = fake_slurm_step_layout_create( | |
| step_node_list, NULL, NULL, node_count, node_count, | |
| SLURM_PROTOCOL_VERSION); | |
| xfree(step_node_list); | |
| if (!step_ptr->step_layout) { | |
| delete_step_record(job_ptr, step_ptr); | |
| return SLURM_ERROR; | |
| } | |
| /* Needed for not considering it in _mark_busy_nodes */ | |
| step_ptr->flags |= SSF_EXT_LAUNCHER; | |
| if ((rc = _set_step_id(step_ptr, step_specs))) { | |
| delete_step_record(job_ptr, step_ptr); | |
| return rc; | |
| } | |
| /* The step needs to run on all the cores. */ | |
| step_ptr->core_bitmap_job = bit_copy(job_ptr->job_resrcs->core_bitmap); | |
| step_ptr->name = xstrdup(step_specs->name); | |
| step_ptr->state = JOB_RUNNING; | |
| step_ptr->start_time = job_ptr->start_time; | |
| step_ptr->time_last_active = time(NULL); | |
| step_set_alloc_tres(step_ptr, 1, false, false); | |
| jobacct_storage_g_step_start(stepmgr_ops->acct_db_conn, step_ptr); | |
| if ((rc = _switch_setup(step_ptr))) { | |
| delete_step_record(job_ptr, step_ptr); | |
| return rc; | |
| } | |
| return SLURM_SUCCESS; | |
| } | |
| extern slurm_node_alias_addrs_t *build_alias_addrs(job_record_t *job_ptr) | |
| { | |
| slurm_node_alias_addrs_t *alias_addrs; | |
| if (!job_ptr || !job_ptr->node_addrs) | |
| return NULL; | |
| alias_addrs = xmalloc(sizeof(slurm_node_alias_addrs_t)); | |
| alias_addrs->node_cnt = job_ptr->node_cnt; | |
| alias_addrs->node_addrs = xcalloc(job_ptr->node_cnt, | |
| sizeof(slurm_addr_t)); | |
| memcpy(alias_addrs->node_addrs, job_ptr->node_addrs, | |
| (sizeof(slurm_addr_t) * job_ptr->node_cnt)); | |
| alias_addrs->node_list = xstrdup(job_ptr->nodes); | |
| alias_addrs->resolve_nodes = xstrdup(job_ptr->resolve_nodes); | |
| return alias_addrs; | |
| } | |
| extern int job_get_node_inx(char *node_name, bitstr_t *node_bitmap) | |
| { | |
| int node_inx = -1; | |
| if (!node_name) | |
| return -1; | |
| xassert(node_bitmap); | |
| node_inx = node_name_get_inx(node_name); | |
| if (node_inx == -1) | |
| return -1; | |
| if (!bit_test(node_bitmap, node_inx)) | |
| return -1; | |
| return bit_set_count_range(node_bitmap, 0, node_inx); | |
| } | |
| static void _kill_step_on_msg_fail(step_complete_msg_t *req, slurm_msg_t *msg, | |
| void (*lock_func)(bool lock)) | |
| { | |
| int rc, rem; | |
| uint32_t step_rc; | |
| DEF_TIMERS; | |
| /* init */ | |
| START_TIMER; | |
| error("Step creation timed out: Deallocating %ps nodes %u-%u", | |
| &req->step_id, req->range_first, req->range_last); | |
| if (lock_func) | |
| lock_func(true); | |
| rc = step_partial_comp(msg, req, true, &rem, &step_rc); | |
| if (lock_func) | |
| lock_func(false); | |
| END_TIMER2(__func__); | |
| log_flag(STEPS, "%s: %ps rc:%s %s", | |
| __func__, &req->step_id, slurm_strerror(rc), TIMER_STR()); | |
| } | |
| /* create a credential for a given job step, return error code */ | |
| static int _make_step_cred(step_record_t *step_ptr, slurm_cred_t **slurm_cred, | |
| uint16_t protocol_version) | |
| { | |
| slurm_cred_arg_t cred_arg; | |
| job_record_t *job_ptr = step_ptr->job_ptr; | |
| job_resources_t *job_resrcs_ptr = job_ptr->job_resrcs; | |
| xassert(job_resrcs_ptr && job_resrcs_ptr->cpus); | |
| setup_cred_arg(&cred_arg, job_ptr); | |
| memcpy(&cred_arg.step_id, &step_ptr->step_id, sizeof(cred_arg.step_id)); | |
| if (job_resrcs_ptr->memory_allocated) { | |
| slurm_array64_to_value_reps(job_resrcs_ptr->memory_allocated, | |
| job_resrcs_ptr->nhosts, | |
| &cred_arg.job_mem_alloc, | |
| &cred_arg.job_mem_alloc_rep_count, | |
| &cred_arg.job_mem_alloc_size); | |
| } | |
| cred_arg.step_gres_list = step_ptr->gres_list_alloc; | |
| cred_arg.step_core_bitmap = step_ptr->core_bitmap_job; | |
| cred_arg.step_hostlist = step_ptr->step_layout->node_list; | |
| if (step_ptr->memory_allocated) { | |
| slurm_array64_to_value_reps(step_ptr->memory_allocated, | |
| step_ptr->step_layout->node_cnt, | |
| &cred_arg.step_mem_alloc, | |
| &cred_arg.step_mem_alloc_rep_count, | |
| &cred_arg.step_mem_alloc_size); | |
| } | |
| cred_arg.switch_step = step_ptr->switch_step; | |
| *slurm_cred = slurm_cred_job_create(&cred_arg, job_ptr, true, | |
| protocol_version); | |
| xfree(cred_arg.job_mem_alloc); | |
| xfree(cred_arg.job_mem_alloc_rep_count); | |
| xfree(cred_arg.step_mem_alloc); | |
| xfree(cred_arg.step_mem_alloc_rep_count); | |
| if (*slurm_cred == NULL) { | |
| error("slurm_cred_create error"); | |
| return ESLURM_INVALID_JOB_CREDENTIAL; | |
| } | |
| return SLURM_SUCCESS; | |
| } | |
| static int _send_msg(slurm_msg_t *msg, int slurmd_fd, slurm_msg_type_t type, | |
| void *data) | |
| { | |
| xassert(running_in_slurmctld() || running_in_slurmstepd()); | |
| int rc; | |
| if (running_in_slurmctld()) { | |
| if ((rc = send_msg_response(msg, type, data))) { | |
| errno = rc; | |
| return SLURM_ERROR; | |
| } | |
| return SLURM_SUCCESS; | |
| } | |
| if (running_in_slurmstepd()) { | |
| xassert(slurmd_fd != -1); | |
| if ((stepd_proxy_send_resp_to_slurmd(slurmd_fd, msg, type, | |
| data))) { | |
| return SLURM_ERROR; | |
| } | |
| return SLURM_SUCCESS; | |
| } | |
| return SLURM_ERROR; | |
| } | |
| extern int step_create_from_msg(slurm_msg_t *msg, int slurmd_fd, | |
| void (*lock_func)(bool lock), | |
| void (*fail_lock_func)(bool lock)) | |
| { | |
| char *err_msg = NULL; | |
| int error_code = SLURM_SUCCESS; | |
| DEF_TIMERS; | |
| step_record_t *step_rec; | |
| step_create_resp_msg_t job_step_resp; | |
| step_create_req_msg_t *req_step_msg = msg->data; | |
| slurm_cred_t *slurm_cred = NULL; | |
| job_record_t *job_ptr = NULL; | |
| bool wake_pending = false; | |
| START_TIMER; | |
| xassert(msg->auth.ids_set); | |
| _dump_step_desc(req_step_msg); | |
| if (lock_func) { | |
| lock_func(true); | |
| } | |
| job_ptr = _find_step_req_job(&req_step_msg->step_id, | |
| req_step_msg->array_task_id); | |
| if (job_ptr == NULL) { | |
| error_code = ESLURM_INVALID_JOB_ID ; | |
| goto end_it; | |
| } | |
| if (job_ptr->bit_flags & EXTERNAL_JOB) { | |
| error("%s: step creation disabled for external jobs", __func__); | |
| error_code = ESLURM_NOT_SUPPORTED; | |
| goto end_it; | |
| } | |
| /* | |
| * Refuse to create steps for pre-25.11 client commands. | |
| * Older clients may not relay the SLUID correctly, which precludes | |
| * safely relaunching the job immediately. | |
| */ | |
| if ((job_ptr->bit_flags & EXPEDITED_REQUEUE) && | |
| (msg->protocol_version < SLURM_25_11_PROTOCOL_VERSION)) { | |
| error("%s: step creation disabled for expedited requeue %pJ for pre-25.11 clients", | |
| __func__, job_ptr); | |
| error_code = ESLURM_NOT_SUPPORTED; | |
| goto end_it; | |
| } | |
| if (running_in_slurmctld() && | |
| (job_ptr->bit_flags & STEPMGR_ENABLED)) { | |
| reroute_msg_t reroute_msg = { | |
| .stepmgr = job_ptr->batch_host, | |
| }; | |
| _send_msg(msg, slurmd_fd, RESPONSE_SLURM_REROUTE_MSG, | |
| &reroute_msg); | |
| if (lock_func) | |
| lock_func(false); | |
| return SLURM_SUCCESS; | |
| } | |
| /* Direct RPC clients must satisfy the same restrictions as srun. */ | |
| if (req_step_msg->step_dependency) { | |
| if (running_in_slurmctld()) { | |
| error("%s: step dependencies require a job with stepmgr enabled", | |
| __func__); | |
| error_code = ESLURM_NOT_SUPPORTED; | |
| goto end_it; | |
| } | |
| if (job_ptr->het_job_id) { | |
| error("%s: step dependencies are not supported for het jobs", | |
| __func__); | |
| error_code = ESLURM_NOT_SUPPORTED; | |
| goto end_it; | |
| } | |
| if (msg->protocol_version < SLURM_26_11_PROTOCOL_VERSION) { | |
| error("%s: step dependencies require a 26.11 client", | |
| __func__); | |
| error_code = ESLURM_NOT_SUPPORTED; | |
| goto end_it; | |
| } | |
| } | |
| if ((req_step_msg->flags & SSF_ASYNC) && | |
| (running_in_slurmctld() || req_step_msg->immediate || | |
| !req_step_msg->launch_params || req_step_msg->launch_params->pty || | |
| job_ptr->het_job_id || | |
| !mpi_is_none_plugin(req_step_msg->launch_params | |
| ->mpi_plugin_name))) { | |
| error("Invalid async step create request"); | |
| error_code = ESLURM_INVALID_FEATURE; | |
| goto end_it; | |
| } | |
| step_rec = NULL; | |
| if (!(req_step_msg->flags & SSF_ASYNC) && | |
| (req_step_msg->step_id.step_id != NO_VAL)) | |
| step_rec = find_step_record(job_ptr, &req_step_msg->step_id); | |
| if (step_rec && | |
| (IS_STEP_CONFIGURING(step_rec) || | |
| (IS_STEP_RUNNING(step_rec) && !IS_STEP_COMPLETING(step_rec)))) { | |
| /* | |
| * The reconcile below skips _step_create(), so apply the | |
| * job-state gates it would have. slurmd's RELAY_AUTH_JOB | |
| * already bound this RPC to the job's own user, so the | |
| * requester check is a belt-and-braces repeat of that rather | |
| * than the gate doing the work. | |
| */ | |
| if (!validate_job_owner(msg, job_ptr)) { | |
| error_code = ESLURM_ACCESS_DENIED; | |
| goto end_it; | |
| } | |
| /* | |
| * Only the srun that owns the step may reconcile it. A | |
| * re-send comes from the same process on the same node, so | |
| * anything else is a second srun of the same user naming an | |
| * id it does not own. | |
| */ | |
| if ((step_rec->srun_pid != req_step_msg->srun_pid) || | |
| xstrcmp(step_rec->host, req_step_msg->host)) { | |
| error_code = ESLURM_DUPLICATE_STEP_ID; | |
| goto end_it; | |
| } | |
| error_code = _job_can_run_step(job_ptr); | |
| if (error_code) | |
| goto end_it; | |
| /* | |
| * A cancel landed while the push was in flight and srun was | |
| * told; re-sending means it never saw that. Do not hand back | |
| * a step the user has already cancelled. | |
| */ | |
| if (step_rec->kill_user) { | |
| error_code = ESLURM_STEP_CANCELLED; | |
| goto end_it; | |
| } | |
| /* | |
| * Lost-push re-send: the step was created at wake and pushed, | |
| * but srun never got the push and re-sent; or the push was | |
| * delivered, clearing the provisional mark, while a re-send was | |
| * already in flight. Re-sending proves srun is alive, so clear | |
| * the provisional mark and re-deliver the existing step (a fresh | |
| * cred + its existing layout) rather than create a duplicate | |
| * (ESLURM_DUPLICATE_STEP_ID). | |
| */ | |
| step_rec->state &= ~JOB_CONFIGURING; | |
| error_code = _make_step_cred(step_rec, &slurm_cred, | |
| step_rec->start_protocol_ver); | |
| if (error_code) { | |
| wake_pending |= _cleanup_failed_step(step_rec); | |
| step_rec = NULL; | |
| } | |
| } else { | |
| step_rec = NULL; | |
| error_code = _step_create(msg, job_ptr, req_step_msg, &step_rec, | |
| NULL, &wake_pending, &err_msg); | |
| if (error_code == SLURM_SUCCESS) { | |
| error_code = | |
| _make_step_cred(step_rec, &slurm_cred, | |
| step_rec->start_protocol_ver); | |
| if (error_code) { | |
| wake_pending |= _cleanup_failed_step(step_rec); | |
| step_rec = NULL; | |
| } | |
| } | |
| } | |
| END_TIMER2(__func__); | |
| end_it: | |
| /* return result */ | |
| if (error_code) { | |
| if (error_code == ESLURM_STEP_QUEUED) { | |
| step_record_t *pending_step = NULL; | |
| pending_step = find_step_record(job_ptr, | |
| &req_step_msg->step_id); | |
| if (pending_step && | |
| (pending_step->step_req == req_step_msg)) | |
| msg->data = NULL; | |
| } | |
| /* A failed re-send may have published a terminal outcome. */ | |
| if (wake_pending) | |
| _wake_pending_steps(job_ptr); | |
| if (lock_func) | |
| lock_func(false); | |
| if (error_code == ESLURM_PROLOG_RUNNING) | |
| log_flag(STEPS, "%s for configuring %ps: %s", | |
| __func__, &req_step_msg->step_id, | |
| slurm_strerror(error_code)); | |
| else if (error_code == ESLURM_DISABLED) | |
| log_flag(STEPS, "%s for suspended %ps: %s", | |
| __func__, &req_step_msg->step_id, | |
| slurm_strerror(error_code)); | |
| else if (error_code == ESLURM_STEP_QUEUED) { | |
| log_flag(STEPS, "%s queued %ps: %s", | |
| __func__, &req_step_msg->step_id, | |
| slurm_strerror(error_code)); | |
| /* | |
| * The placeholder retains step_req for create-at-wake | |
| * (async always; a sync step only in the stepmgr, and | |
| * only for a >= 26.11 srun that can consume the push), | |
| * so stop the regular RPC flow from freeing it. Reply | |
| * with | |
| * RESPONSE_JOB_STEP_CREATE carrying the assigned | |
| * step_id and state = JOB_PENDING so srun can surface | |
| * it. _queue_pending_step() reports queued only once | |
| * a placeholder has an assigned step_id. Only a newly | |
| * retained request was detached above; a re-send's | |
| * request remains owned by this RPC. | |
| */ | |
| memset(&job_step_resp, 0, sizeof(job_step_resp)); | |
| job_step_resp.step_id = req_step_msg->step_id; | |
| job_step_resp.state = JOB_PENDING; | |
| _send_msg(msg, slurmd_fd, RESPONSE_JOB_STEP_CREATE, | |
| &job_step_resp); | |
| return error_code; | |
| } else | |
| log_flag(STEPS, "%s for %ps: %s", | |
| __func__, &req_step_msg->step_id, | |
| slurm_strerror(error_code)); | |
| if (err_msg) { | |
| return_code2_msg_t rc_msg = { | |
| .return_code = error_code, | |
| .err_msg = err_msg, | |
| }; | |
| _send_msg(msg, slurmd_fd, RESPONSE_SLURM_RC_MSG, | |
| &rc_msg); | |
| } else { | |
| return_code_msg_t rc_msg = { | |
| .return_code = error_code, | |
| }; | |
| _send_msg(msg, slurmd_fd, RESPONSE_SLURM_RC, &rc_msg); | |
| } | |
| } else { | |
| slurm_step_layout_t *step_layout = NULL; | |
| log_flag(STEPS, "%s: %pS %s %s", | |
| __func__, step_rec, step_rec->step_layout->node_list, | |
| TIMER_STR()); | |
| memset(&job_step_resp, 0, sizeof(job_step_resp)); | |
| job_step_resp.step_id = step_rec->step_id; | |
| job_step_resp.state = JOB_RUNNING; | |
| job_step_resp.resv_ports = step_rec->resv_ports; | |
| step_layout = slurm_step_layout_copy(step_rec->step_layout); | |
| job_step_resp.step_layout = step_layout; | |
| if (step_rec->job_ptr && step_rec->job_ptr->details && | |
| (step_rec->job_ptr->details->cpu_bind_type != NO_VAL16)) { | |
| job_step_resp.def_cpu_bind_type = | |
| step_rec->job_ptr->details->cpu_bind_type; | |
| } | |
| job_step_resp.cred = slurm_cred; | |
| job_step_resp.use_protocol_ver = step_rec->start_protocol_ver; | |
| if (job_ptr->bit_flags & STEPMGR_ENABLED) | |
| job_step_resp.stepmgr = job_ptr->batch_host; | |
| /* | |
| * This step starting is what an "after" dependent was waiting | |
| * for, so release any such step before dropping the lock. The | |
| * response above was built first: the drain may complete and | |
| * free other steps, and an impossible dependent is finished | |
| * here rather than left queued. | |
| */ | |
| if (wake_pending) | |
| _wake_pending_steps(job_ptr); | |
| if (lock_func) | |
| lock_func(false); | |
| if (msg->protocol_version != step_rec->start_protocol_ver) { | |
| log_flag(NET, "%s: responding with non-matching msg 0x%x to step 0x%x protocol version", | |
| __func__, msg->protocol_version, | |
| step_rec->start_protocol_ver); | |
| msg->protocol_version = step_rec->start_protocol_ver; | |
| } | |
| if ((error_code == SLURM_SUCCESS) && | |
| (step_rec->flags & SSF_ASYNC)) { | |
| slurm_step_ctx_t *ctx; | |
| debug2("launching async step"); | |
| req_step_msg->use_protocol_ver = | |
| step_rec->start_protocol_ver; | |
| /* | |
| * Set to "none" when not set, to prevent | |
| * mpi_g_client_init() setting to MpiDefault. | |
| */ | |
| if (!req_step_msg->launch_params->mpi_plugin_name) | |
| req_step_msg->launch_params->mpi_plugin_name = | |
| xstrdup("none"); | |
| ctx = _step_ctx_create_stepmgr(req_step_msg, | |
| &job_step_resp, | |
| step_rec); | |
| error_code = slurm_step_launch( | |
| ctx, req_step_msg->launch_params, NULL); | |
| ctx->step_req = NULL; | |
| ctx->step_resp = NULL; | |
| step_ctx_destroy(ctx); | |
| if (error_code != SLURM_SUCCESS) { | |
| error("Could not launch async step: %s", | |
| slurm_strerror(error_code)); | |
| if (_cleanup_failed_step(step_rec)) | |
| _wake_pending_steps(job_ptr); | |
| step_rec = NULL; | |
| } | |
| } | |
| if (error_code == SLURM_SUCCESS) { | |
| if (_send_msg(msg, slurmd_fd, | |
| RESPONSE_JOB_STEP_CREATE, | |
| &job_step_resp)) { | |
| step_complete_msg_t req; | |
| memset(&req, 0, sizeof(req)); | |
| req.step_id = step_rec->step_id; | |
| req.jobacct = step_rec->jobacct; | |
| req.step_rc = SIGKILL; | |
| req.range_first = 0; | |
| req.range_last = step_layout->node_cnt - 1; | |
| _kill_step_on_msg_fail(&req, msg, | |
| fail_lock_func); | |
| } | |
| } else { | |
| return_code_msg_t rc_msg = { | |
| .return_code = error_code, | |
| }; | |
| _send_msg(msg, slurmd_fd, RESPONSE_SLURM_RC, | |
| &rc_msg); | |
| } | |
| slurm_cred_destroy(slurm_cred); | |
| slurm_step_layout_destroy(step_layout); | |
| } | |
| xfree(err_msg); | |
| return error_code; | |
| } | |
| /* | |
| * pack_job_step_info_response_msg - packs job step info | |
| * IN step_id - specific id or NO_VAL/NO_VAL for all | |
| * IN uid - user issuing request | |
| * IN show_flags - job step filtering options | |
| * OUT buffer - location to store data, pointers automatically advanced | |
| * RET - 0 or error code | |
| * NOTE: MUST free_buf buffer | |
| */ | |
| extern int pack_job_step_info_response_msg(pack_step_args_t *args) | |
| { | |
| int error_code = 0; | |
| uint32_t tmp_offset; | |
| time_t now = time(NULL); | |
| if (args->proto_version >= SLURM_MIN_PROTOCOL_VERSION) { | |
| /* steps_packed placeholder */ | |
| pack32(args->steps_packed, args->buffer); | |
| pack_time(now, args->buffer); | |
| list_for_each_ro(args->job_step_list, | |
| args->pack_job_step_list_func, args); | |
| if (list_count(job_list) && !args->valid_job && | |
| !args->steps_packed) | |
| error_code = ESLURM_INVALID_JOB_ID; | |
| slurm_pack_list(args->stepmgr_jobs, | |
| slurm_pack_stepmgr_job_info, args->buffer, | |
| args->proto_version); | |
| /* put the real record count in the message body header */ | |
| tmp_offset = get_buf_offset(args->buffer); | |
| set_buf_offset(args->buffer, 0); | |
| pack32(args->steps_packed, args->buffer); | |
| set_buf_offset(args->buffer, tmp_offset); | |
| } | |
| FREE_NULL_LIST(args->stepmgr_jobs); | |
| xfree(args->visible_parts); | |
| return error_code; | |
| } | |
| extern int stepmgr_get_step_layouts(job_record_t *job_ptr, | |
| slurm_step_id_t *step_id, | |
| slurm_step_layout_t **out_step_layout) | |
| { | |
| list_itr_t *itr; | |
| step_record_t *step_ptr = NULL; | |
| slurm_step_layout_t *step_layout = NULL; | |
| /* We can't call find_step_record here since we may need more than 1 */ | |
| itr = list_iterator_create(job_ptr->step_list); | |
| while ((step_ptr = list_next(itr))) { | |
| if (!verify_step_id(&step_ptr->step_id, step_id)) | |
| continue; | |
| /* A queued placeholder has no layout yet */ | |
| if (!step_ptr->step_layout) | |
| continue; | |
| /* | |
| * Rebuild alias_addrs if need after restart of slurmctld | |
| */ | |
| if (job_ptr->node_addrs && | |
| !step_ptr->step_layout->alias_addrs) { | |
| step_ptr->step_layout->alias_addrs = | |
| build_alias_addrs(job_ptr); | |
| } | |
| if (step_layout) | |
| slurm_step_layout_merge(step_layout, | |
| step_ptr->step_layout); | |
| else | |
| step_layout = slurm_step_layout_copy( | |
| step_ptr->step_layout); | |
| /* break if don't need to look for further het_steps */ | |
| if (step_ptr->step_id.step_het_comp == NO_VAL) | |
| break; | |
| /* | |
| * If we are looking for a specific het step we can break here | |
| * as well. | |
| */ | |
| if (step_id->step_het_comp != NO_VAL) | |
| break; | |
| } | |
| list_iterator_destroy(itr); | |
| if (!step_layout) { | |
| log_flag(STEPS, "%s: %pJ StepId=%u Not Found", | |
| __func__, job_ptr, step_id->step_id); | |
| return ESLURM_INVALID_JOB_ID; | |
| } | |
| /* | |
| * The cpt_compact* fields don't go to the client because they are not | |
| * handled in slurm_step_layout_merge(). Free them so the client does | |
| * not get bad data. | |
| */ | |
| xfree(step_layout->cpt_compact_array); | |
| xfree(step_layout->cpt_compact_reps); | |
| step_layout->cpt_compact_cnt = 0; | |
| *out_step_layout = step_layout; | |
| return SLURM_SUCCESS; | |
| } | |
| extern int stepmgr_get_job_sbcast_cred_msg(job_record_t *job_ptr, | |
| slurm_step_id_t *step_id, | |
| uint16_t protocol_version, | |
| job_sbcast_cred_msg_t **out_sbcast_cred_msg) | |
| { | |
| sbcast_cred_t *sbcast_cred; | |
| sbcast_cred_arg_t sbcast_arg; | |
| step_record_t *step_ptr = NULL; | |
| char *node_list = NULL; | |
| job_sbcast_cred_msg_t *job_info_resp_msg; | |
| xassert(job_ptr); | |
| /* | |
| * Refuse to create credentials for pre-25.11 client commands. | |
| * Older clients may not relay the SLUID correctly, which precludes | |
| * safely relaunching the job immediately. | |
| */ | |
| if ((job_ptr->bit_flags & EXPEDITED_REQUEUE) && | |
| (protocol_version < SLURM_25_11_PROTOCOL_VERSION)) { | |
| error("%s: sbcast cred creation disabled for expedited requeue %pJ for pre-25.11 clients", | |
| __func__, job_ptr); | |
| return ESLURM_NOT_SUPPORTED; | |
| } | |
| if (step_id->step_id != NO_VAL) { | |
| step_ptr = find_step_record(job_ptr, step_id); | |
| if (!step_ptr) { | |
| return ESLURM_INVALID_JOB_ID; | |
| } else if (step_ptr->step_layout && | |
| (step_ptr->step_layout->node_cnt != | |
| job_ptr->node_cnt)) { | |
| node_list = step_ptr->step_layout->node_list; | |
| } | |
| } | |
| if (!node_list) | |
| node_list = job_ptr->nodes; | |
| /* | |
| * Note - using pointers to other xmalloc'd elements owned by other | |
| * structures to avoid copy overhead. Do not free them! | |
| */ | |
| memset(&sbcast_arg, 0, sizeof(sbcast_arg)); | |
| sbcast_arg.step_id = STEP_ID_FROM_JOB_RECORD(job_ptr); | |
| sbcast_arg.het_job_id = job_ptr->het_job_id; | |
| if (step_ptr) | |
| sbcast_arg.step_id.step_id = step_ptr->step_id.step_id; | |
| else | |
| sbcast_arg.step_id.step_id = job_ptr->next_step_id; | |
| sbcast_arg.nodes = node_list; /* avoid extra copy */ | |
| sbcast_arg.expiration = job_ptr->end_time; | |
| sbcast_arg.id = job_ptr->id; | |
| if (!(sbcast_cred = create_sbcast_cred(&sbcast_arg, job_ptr->user_id, | |
| job_ptr->group_id, | |
| protocol_version))) { | |
| error("%s %pJ cred create error", __func__, job_ptr); | |
| return SLURM_ERROR; | |
| } | |
| job_info_resp_msg = xmalloc(sizeof(*job_info_resp_msg)); | |
| job_info_resp_msg->node_list = xstrdup(node_list); | |
| job_info_resp_msg->sbcast_cred = sbcast_cred; | |
| *out_sbcast_cred_msg = job_info_resp_msg; | |
| return SLURM_SUCCESS; | |
| } | |
| /* Build structure with job allocation details */ | |
| extern resource_allocation_response_msg_t *build_job_info_resp( | |
| job_record_t *job_ptr) | |
| { | |
| resource_allocation_response_msg_t *job_info_resp_msg; | |
| int i; | |
| job_info_resp_msg = xmalloc(sizeof(resource_allocation_response_msg_t)); | |
| job_info_resp_msg->account = xstrdup(job_ptr->account); | |
| job_info_resp_msg->batch_host = xstrdup(job_ptr->batch_host); | |
| job_info_resp_msg->step_id = STEP_ID_FROM_JOB_RECORD(job_ptr); | |
| job_info_resp_msg->node_cnt = job_ptr->node_cnt; | |
| if (job_ptr->job_resrcs) { | |
| job_emit_node_arrays(job_ptr, &job_info_resp_msg->node_list, | |
| &job_info_resp_msg->cpus_per_node, | |
| &job_info_resp_msg->cpu_count_reps, | |
| &job_info_resp_msg->num_cpu_groups); | |
| } else { | |
| job_info_resp_msg->node_list = xstrdup(job_ptr->nodes); | |
| } | |
| if (job_ptr->part_ptr) | |
| job_info_resp_msg->partition = xstrdup(job_ptr->part_ptr->name); | |
| else | |
| job_info_resp_msg->partition = xstrdup(job_ptr->partition); | |
| if (job_ptr->qos_ptr) { | |
| slurmdb_qos_rec_t *qos; | |
| qos = (slurmdb_qos_rec_t *)job_ptr->qos_ptr; | |
| job_info_resp_msg->qos = xstrdup(qos->name); | |
| } | |
| job_info_resp_msg->resv_name = xstrdup(job_ptr->resv_name); | |
| if (job_ptr->bit_flags & STEPMGR_ENABLED) | |
| job_info_resp_msg->stepmgr_host = xstrdup(job_ptr->batch_host); | |
| if (job_ptr->details) { | |
| if (job_ptr->bit_flags & JOB_MEM_SET) { | |
| job_info_resp_msg->pn_min_memory = | |
| job_ptr->details->pn_min_memory; | |
| } | |
| if (job_ptr->details->mc_ptr) { | |
| job_info_resp_msg->ntasks_per_board = | |
| job_ptr->details->mc_ptr->ntasks_per_board; | |
| job_info_resp_msg->ntasks_per_core = | |
| job_ptr->details->mc_ptr->ntasks_per_core; | |
| job_info_resp_msg->ntasks_per_socket = | |
| job_ptr->details->mc_ptr->ntasks_per_socket; | |
| } | |
| } else { | |
| /* job_info_resp_msg->pn_min_memory = 0; */ | |
| job_info_resp_msg->ntasks_per_board = NO_VAL16; | |
| job_info_resp_msg->ntasks_per_core = NO_VAL16; | |
| job_info_resp_msg->ntasks_per_socket = NO_VAL16; | |
| } | |
| if (job_ptr->details && job_ptr->details->env_cnt) { | |
| job_info_resp_msg->env_size = job_ptr->details->env_cnt; | |
| job_info_resp_msg->environment = | |
| xcalloc(job_info_resp_msg->env_size + 1, | |
| sizeof(char *)); | |
| for (i = 0; i < job_info_resp_msg->env_size; i++) { | |
| job_info_resp_msg->environment[i] = | |
| xstrdup(job_ptr->details->env_sup[i]); | |
| } | |
| job_info_resp_msg->environment[i] = NULL; | |
| } | |
| job_info_resp_msg->uid = job_ptr->user_id; | |
| job_info_resp_msg->user_name = user_from_job(job_ptr); | |
| job_info_resp_msg->gid = job_ptr->group_id; | |
| job_info_resp_msg->group_name = group_from_job(job_ptr); | |
| return job_info_resp_msg; | |
| } |