| /****************************************************************************\ |
| * statistics.c - functions for sdiag command |
| ***************************************************************************** |
| * Produced at Barcelona Supercomputing Center, December 2011 |
| * Written by Alejandro Lucero <alucero@bsc.es> |
| * |
| * This file is part of Slurm, a resource management program. |
| * For details, see <https://slurm.schedmd.com/>. |
| * Please also read the included file: DISCLAIMER. |
| * |
| * Slurm is free software; you can redistribute it and/or modify it under |
| * the terms of the GNU General Public License as published by the Free |
| * Software Foundation; either version 2 of the License, or (at your option) |
| * any later version. |
| * |
| * In addition, as a special exception, the copyright holders give permission |
| * to link the code of portions of this program with the OpenSSL library under |
| * certain conditions as described in each individual source file, and |
| * distribute linked combinations including the two. You must obey the GNU |
| * General Public License in all respects for all of the code used other than |
| * OpenSSL. If you modify file(s) with this exception, you may extend this |
| * exception to your version of the file(s), but you are not obligated to do |
| * so. If you do not wish to do so, delete this exception statement from your |
| * version. If you delete this exception statement from all source files in |
| * the program, then also delete it here. |
| * |
| * Slurm is distributed in the hope that it will be useful, but WITHOUT ANY |
| * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
| * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more |
| * details. |
| * |
| * You should have received a copy of the GNU General Public License along |
| * with Slurm; if not, write to the Free Software Foundation, Inc., |
| * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
| \*****************************************************************************/ |
| |
| #include <ctype.h> |
| #include <errno.h> |
| #include <stdio.h> |
| |
| #include "src/slurmctld/agent.h" |
| #include "src/slurmctld/slurmctld.h" |
| #include "src/common/id_util.h" |
| #include "src/common/list.h" |
| #include "src/common/pack.h" |
| #include "src/common/xstring.h" |
| #include "src/common/slurmdbd_defs.h" |
| |
| #include "src/common/assoc_mgr.h" |
| #include "src/interfaces/gres.h" |
| #include "src/interfaces/select.h" |
| |
| #include "src/slurmctld/locks.h" |
| #include "src/slurmctld/statistics.h" |
| |
| typedef struct foreach_fill_jobs_args { |
| int gpu_tres_pos; |
| jobs_stats_t *js; |
| } foreach_fill_jobs_args_t; |
| |
| typedef struct foreach_part_gen_stats { |
| jobs_stats_t *js; |
| nodes_stats_t *ns; |
| partitions_stats_t *ps; |
| } foreach_part_gen_stats_t; |
| |
| /* Pack all scheduling statistics */ |
| extern buf_t *pack_all_stat(uint16_t protocol_version) |
| { |
| buf_t *buffer; |
| int agent_queue_size; |
| int agent_count; |
| int agent_thread_count; |
| int slurmdbd_queue_size = 0; |
| time_t now = time(NULL); |
| |
| if (acct_storage_g_get_data(acct_db_conn, ACCT_STORAGE_INFO_AGENT_COUNT, |
| &slurmdbd_queue_size) != SLURM_SUCCESS) |
| slurmdbd_queue_size = 0; |
| |
| buffer = init_buf(BUF_SIZE); |
| if (protocol_version >= SLURM_25_11_PROTOCOL_VERSION) { |
| pack_time(now, buffer); |
| debug3("%s: time = %lu", __func__, last_proc_req_start); |
| pack_time(last_proc_req_start, buffer); |
| |
| slurm_mutex_lock(&slurmctld_config.thread_count_lock); |
| debug3("%s: server_thread_count = %u", |
| __func__, slurmctld_config.server_thread_count); |
| pack32(slurmctld_config.server_thread_count, buffer); |
| slurm_mutex_unlock(&slurmctld_config.thread_count_lock); |
| |
| agent_queue_size = retry_list_size(); |
| pack32(agent_queue_size, buffer); |
| agent_count = get_agent_count(); |
| pack32(agent_count, buffer); |
| agent_thread_count = get_agent_thread_count(); |
| pack32(agent_thread_count, buffer); |
| pack32(slurmdbd_queue_size, buffer); |
| pack32(slurmctld_diag_stats.latency, buffer); |
| |
| pack32(slurmctld_diag_stats.jobs_submitted, buffer); |
| pack32(slurmctld_diag_stats.jobs_started, buffer); |
| pack32(slurmctld_diag_stats.jobs_completed, buffer); |
| pack32(slurmctld_diag_stats.jobs_canceled, buffer); |
| pack32(slurmctld_diag_stats.jobs_failed, buffer); |
| |
| pack32(slurmctld_diag_stats.jobs_pending, buffer); |
| pack32(slurmctld_diag_stats.jobs_running, buffer); |
| pack_time(slurmctld_diag_stats.job_states_ts, buffer); |
| |
| pack32(slurmctld_diag_stats.schedule_cycle_max, buffer); |
| pack32(slurmctld_diag_stats.schedule_cycle_last, buffer); |
| pack64(slurmctld_diag_stats.schedule_cycle_sum, buffer); |
| pack32(slurmctld_diag_stats.schedule_cycle_counter, buffer); |
| pack32(slurmctld_diag_stats.schedule_cycle_depth, buffer); |
| pack32_array(slurmctld_diag_stats.schedule_exit, |
| SCHEDULE_EXIT_COUNT, buffer); |
| pack32(slurmctld_diag_stats.schedule_queue_len, buffer); |
| |
| pack32(slurmctld_diag_stats.backfilled_jobs, buffer); |
| pack32(slurmctld_diag_stats.last_backfilled_jobs, buffer); |
| pack32(slurmctld_diag_stats.bf_cycle_counter, buffer); |
| pack64(slurmctld_diag_stats.bf_cycle_sum, buffer); |
| pack32(slurmctld_diag_stats.bf_cycle_last, buffer); |
| pack32(slurmctld_diag_stats.bf_last_depth, buffer); |
| pack32(slurmctld_diag_stats.bf_last_depth_try, buffer); |
| |
| pack32(slurmctld_diag_stats.bf_queue_len, buffer); |
| pack32(slurmctld_diag_stats.bf_cycle_max, buffer); |
| pack_time(slurmctld_diag_stats.bf_when_last_cycle, buffer); |
| pack32(slurmctld_diag_stats.bf_depth_sum, buffer); |
| pack32(slurmctld_diag_stats.bf_depth_try_sum, buffer); |
| pack32(slurmctld_diag_stats.bf_queue_len_sum, buffer); |
| pack32(slurmctld_diag_stats.bf_table_size, buffer); |
| pack32(slurmctld_diag_stats.bf_table_size_sum, buffer); |
| |
| pack32(slurmctld_diag_stats.bf_active, buffer); |
| pack32(slurmctld_diag_stats.backfilled_het_jobs, buffer); |
| pack32_array(slurmctld_diag_stats.bf_exit, BF_EXIT_COUNT, |
| buffer); |
| } else if (protocol_version >= SLURM_MIN_PROTOCOL_VERSION) { |
| pack32(1, buffer); /* please remove on next version */ |
| |
| pack_time(now, buffer); |
| debug3("%s: time = %lu", __func__, last_proc_req_start); |
| pack_time(last_proc_req_start, buffer); |
| |
| slurm_mutex_lock(&slurmctld_config.thread_count_lock); |
| debug3("%s: server_thread_count = %u", |
| __func__, slurmctld_config.server_thread_count); |
| pack32(slurmctld_config.server_thread_count, buffer); |
| slurm_mutex_unlock(&slurmctld_config.thread_count_lock); |
| |
| agent_queue_size = retry_list_size(); |
| pack32(agent_queue_size, buffer); |
| agent_count = get_agent_count(); |
| pack32(agent_count, buffer); |
| agent_thread_count = get_agent_thread_count(); |
| pack32(agent_thread_count, buffer); |
| pack32(slurmdbd_queue_size, buffer); |
| pack32(slurmctld_diag_stats.latency, buffer); |
| |
| pack32(slurmctld_diag_stats.jobs_submitted, buffer); |
| pack32(slurmctld_diag_stats.jobs_started, buffer); |
| pack32(slurmctld_diag_stats.jobs_completed, buffer); |
| pack32(slurmctld_diag_stats.jobs_canceled, buffer); |
| pack32(slurmctld_diag_stats.jobs_failed, buffer); |
| |
| pack32(slurmctld_diag_stats.jobs_pending, buffer); |
| pack32(slurmctld_diag_stats.jobs_running, buffer); |
| pack_time(slurmctld_diag_stats.job_states_ts, buffer); |
| |
| pack32(slurmctld_diag_stats.schedule_cycle_max, buffer); |
| pack32(slurmctld_diag_stats.schedule_cycle_last, buffer); |
| pack32(slurmctld_diag_stats.schedule_cycle_sum, buffer); |
| pack32(slurmctld_diag_stats.schedule_cycle_counter, buffer); |
| pack32(slurmctld_diag_stats.schedule_cycle_depth, buffer); |
| pack32_array(slurmctld_diag_stats.schedule_exit, |
| SCHEDULE_EXIT_COUNT, buffer); |
| pack32(slurmctld_diag_stats.schedule_queue_len, buffer); |
| |
| pack32(slurmctld_diag_stats.backfilled_jobs, buffer); |
| pack32(slurmctld_diag_stats.last_backfilled_jobs, buffer); |
| pack32(slurmctld_diag_stats.bf_cycle_counter, buffer); |
| pack64(slurmctld_diag_stats.bf_cycle_sum, buffer); |
| pack32(slurmctld_diag_stats.bf_cycle_last, buffer); |
| pack32(slurmctld_diag_stats.bf_last_depth, buffer); |
| pack32(slurmctld_diag_stats.bf_last_depth_try, buffer); |
| |
| pack32(slurmctld_diag_stats.bf_queue_len, buffer); |
| pack32(slurmctld_diag_stats.bf_cycle_max, buffer); |
| pack_time(slurmctld_diag_stats.bf_when_last_cycle, buffer); |
| pack32(slurmctld_diag_stats.bf_depth_sum, buffer); |
| pack32(slurmctld_diag_stats.bf_depth_try_sum, buffer); |
| pack32(slurmctld_diag_stats.bf_queue_len_sum, buffer); |
| pack32(slurmctld_diag_stats.bf_table_size, buffer); |
| pack32(slurmctld_diag_stats.bf_table_size_sum, buffer); |
| |
| pack32(slurmctld_diag_stats.bf_active, buffer); |
| pack32(slurmctld_diag_stats.backfilled_het_jobs, buffer); |
| pack32_array(slurmctld_diag_stats.bf_exit, BF_EXIT_COUNT, |
| buffer); |
| } |
| |
| return buffer; |
| } |
| |
| /* Reset all scheduling statistics |
| * level IN - clear backfilled_jobs count if set */ |
| extern void reset_stats(int level) |
| { |
| slurmctld_diag_stats.proc_req_raw = 0; |
| slurmctld_diag_stats.proc_req_threads = 0; |
| slurmctld_diag_stats.schedule_cycle_max = 0; |
| slurmctld_diag_stats.schedule_cycle_sum = 0; |
| slurmctld_diag_stats.schedule_cycle_counter = 0; |
| slurmctld_diag_stats.schedule_cycle_depth = 0; |
| slurmctld_diag_stats.jobs_submitted = 0; |
| slurmctld_diag_stats.jobs_started = 0; |
| slurmctld_diag_stats.jobs_completed = 0; |
| slurmctld_diag_stats.jobs_canceled = 0; |
| slurmctld_diag_stats.jobs_failed = 0; |
| |
| memset(slurmctld_diag_stats.schedule_exit, 0, |
| sizeof(slurmctld_diag_stats.schedule_exit)); |
| |
| /* Just resetting this value when reset requested explicitly */ |
| if (level) |
| slurmctld_diag_stats.backfilled_jobs = 0; |
| |
| slurmctld_diag_stats.last_backfilled_jobs = 0; |
| slurmctld_diag_stats.backfilled_het_jobs = 0; |
| slurmctld_diag_stats.bf_cycle_counter = 0; |
| slurmctld_diag_stats.bf_cycle_sum = 0; |
| slurmctld_diag_stats.bf_cycle_last = 0; |
| slurmctld_diag_stats.bf_depth_sum = 0; |
| slurmctld_diag_stats.bf_depth_try_sum = 0; |
| slurmctld_diag_stats.bf_queue_len = 0; |
| slurmctld_diag_stats.bf_queue_len_sum = 0; |
| slurmctld_diag_stats.bf_table_size_sum = 0; |
| slurmctld_diag_stats.bf_cycle_max = 0; |
| slurmctld_diag_stats.bf_last_depth = 0; |
| slurmctld_diag_stats.bf_last_depth_try = 0; |
| |
| memset(slurmctld_diag_stats.bf_exit, 0, |
| sizeof(slurmctld_diag_stats.bf_exit)); |
| |
| last_proc_req_start = time(NULL); |
| } |
| |
| static void _free_job_stats(job_stats_t *j) |
| { |
| FREE_NULL_BITMAP(j->node_bitmap); |
| xfree(j->user_name); |
| xfree(j->partition); |
| xfree(j->account); |
| xfree(j); |
| } |
| |
| static void _free_part_stats(partition_stats_t *s) |
| { |
| xfree(s->name); |
| xfree(s); |
| } |
| |
| static void _free_ua_stats(ua_stats_t *ua) |
| { |
| statistics_free_jobs(ua->s); |
| xfree(ua->name); |
| xfree(ua); |
| } |
| |
| static int _find_ua_by_name(void *x, void *y) |
| { |
| ua_stats_t *ua = x; |
| |
| if (!xstrcmp(ua->name, (char *) y)) |
| return 1; |
| return 0; |
| } |
| |
| static int _statistics_part_aggregate_job(void *x, void *arg) |
| { |
| partition_stats_t *ps = arg; |
| job_stats_t *j = x; |
| |
| if (xstrcmp(j->partition, ps->name)) |
| return SLURM_SUCCESS; |
| |
| if (IS_JOB_BOOT_FAIL(j)) |
| ps->jobs_bootfail++; |
| if (IS_JOB_CANCELLED(j)) |
| ps->jobs_cancelled++; |
| if (IS_JOB_COMPLETE(j)) |
| ps->jobs_completed++; |
| if (IS_JOB_DEADLINE(j)) |
| ps->jobs_deadline++; |
| if (IS_JOB_FAILED(j)) |
| ps->jobs_failed++; |
| if (IS_JOB_NODE_FAILED(j)) |
| ps->jobs_node_failed++; |
| if (IS_JOB_OOM(j)) |
| ps->jobs_oom++; |
| if (IS_JOB_PENDING(j)) { |
| if ((j->state_reason == WAIT_HELD) || |
| (j->state_reason == WAIT_HELD_USER)) { |
| ps->jobs_hold += j->job_array_cnt; |
| } |
| ps->jobs_pending += j->job_array_cnt; |
| } |
| if (IS_JOB_PREEMPTED(j)) |
| ps->jobs_preempted++; |
| if (IS_JOB_RUNNING(j)) |
| ps->jobs_running++; |
| if (IS_JOB_SUSPENDED(j)) |
| ps->jobs_suspended++; |
| if (IS_JOB_TIMEOUT(j)) |
| ps->jobs_timeout++; |
| |
| /* Derived job states */ |
| if (IS_JOB_COMPLETING(j)) |
| ps->jobs_completing++; |
| if (IS_JOB_CONFIGURING(j)) |
| ps->jobs_configuring++; |
| if (IS_JOB_EXPEDITING(j)) |
| ps->jobs_expediting++; |
| if (IS_JOB_FED_REQUEUED(j)) |
| ps->jobs_fed_requeued++; |
| if (IS_JOB_FINISHED(j)) |
| ps->jobs_finished++; |
| if (IS_JOB_POWER_UP_NODE(j)) |
| ps->jobs_powerup_node++; |
| if (IS_JOB_REQUEUED(j)) |
| ps->jobs_requeued++; |
| if (IS_JOB_RESIZING(j)) |
| ps->jobs_resizing++; |
| if (IS_JOB_REVOKED(j)) |
| ps->jobs_revoked++; |
| if (IS_JOB_SIGNALING(j)) |
| ps->jobs_signaling++; |
| if (IS_JOB_STAGE_OUT(j)) |
| ps->jobs_stageout++; |
| if (IS_JOB_STARTED(j)) |
| ps->jobs_started++; |
| |
| /* Custom metric for Slinky */ |
| if (j->state_reason & WAIT_PART_NODE_LIMIT) |
| ps->jobs_wait_part_node_limit++; |
| |
| if (IS_JOB_RUNNING(j) || IS_JOB_SUSPENDED(j)) { |
| ps->jobs_cpus_alloc += j->cpus_alloc; |
| ps->jobs_gpus_alloc += j->gpus_alloc; |
| ps->jobs_memory_alloc += j->memory_alloc; |
| } |
| |
| if (!IS_JOB_COMPLETED(j)) |
| ps->jobs++; |
| |
| /* |
| * Custom Slinky metrics for autoscaling |
| * Max of the min_nodes required of all pending jobs in that partition |
| * Max of the max_nodes required of all pending jobs in that partition |
| */ |
| if (IS_JOB_PENDING(j)) { |
| if (!((j->state_reason == WAIT_HELD) || |
| (j->state_reason == WAIT_HELD_USER))) { |
| ps->jobs_max_job_nodes_nohold = |
| MAX(ps->jobs_max_job_nodes_nohold, |
| j->max_nodes); |
| ps->jobs_min_job_nodes_nohold = |
| MAX(ps->jobs_min_job_nodes_nohold, |
| j->min_nodes); |
| } |
| ps->jobs_max_job_nodes = |
| MAX(ps->jobs_max_job_nodes, j->max_nodes); |
| ps->jobs_min_job_nodes = |
| MAX(ps->jobs_min_job_nodes, j->min_nodes); |
| } |
| |
| return SLURM_SUCCESS; |
| } |
| |
| static void _statistics_part_aggregate_node(partition_stats_t *ps, |
| node_stats_t *ns) |
| { |
| ps->nodes_cpus_alloc += ns->cpus_alloc; |
| ps->nodes_cpus_efctv += ns->cpus_efctv; |
| ps->nodes_cpus_idle += ns->cpus_idle; |
| |
| ps->total_gpus += ns->gpus_total; |
| |
| ps->nodes_mem_alloc += ns->mem_alloc; |
| ps->nodes_mem_avail += ns->mem_avail; |
| ps->nodes_mem_free += ns->mem_free; |
| ps->nodes_mem_total += ns->mem_total; |
| |
| if (IS_NODE_FUTURE(ns)) |
| ps->nodes_future++; |
| else if (IS_NODE_DOWN(ns)) |
| ps->nodes_down++; |
| else if (IS_NODE_MIXED(ns)) |
| ps->nodes_mixed++; |
| else if (IS_NODE_ALLOCATED(ns)) |
| ps->nodes_alloc++; |
| else if (IS_NODE_IDLE(ns)) |
| ps->nodes_idle++; |
| else if (IS_NODE_UNKNOWN(ns)) |
| ps->nodes_unknown++; |
| |
| /* Derived node states */ |
| if (IS_NODE_COMPLETING(ns)) |
| ps->nodes_cg++; |
| if (IS_NODE_BLOCKED(ns)) |
| ps->nodes_blocked++; |
| if (IS_NODE_CLOUD(ns)) |
| ps->nodes_cloud++; |
| |
| if (IS_NODE_DRAINING(ns)) |
| ps->nodes_draining++; |
| else if (IS_NODE_DRAIN(ns)) |
| ps->nodes_drain++; |
| |
| if (IS_NODE_DRAINED(ns)) |
| ps->nodes_drained++; |
| if (IS_NODE_DYNAMIC_FUTURE(ns)) |
| ps->nodes_dyn_future++; |
| if (IS_NODE_DYNAMIC_NORM(ns)) |
| ps->nodes_dyn_normal++; |
| if (IS_NODE_EXTERNAL(ns)) |
| ps->nodes_external++; |
| if (IS_NODE_FAIL(ns)) |
| ps->nodes_fail++; |
| if (IS_NODE_INVALID_REG(ns)) |
| ps->nodes_invalid_reg++; |
| if (IS_NODE_MAINT(ns)) |
| ps->nodes_maint++; |
| if (IS_NODE_NO_RESPOND(ns)) |
| ps->nodes_no_resp++; |
| if (IS_NODE_PLANNED(ns)) |
| ps->nodes_planned++; |
| if (IS_NODE_POWER_DOWN(ns)) |
| ps->nodes_power_down++; |
| if (IS_NODE_POWER_UP(ns)) |
| ps->nodes_power_up++; |
| if (IS_NODE_POWERED_DOWN(ns)) |
| ps->nodes_powered_down++; |
| if (IS_NODE_POWERING_DOWN(ns)) |
| ps->nodes_powering_down++; |
| if (IS_NODE_POWERING_UP(ns)) |
| ps->nodes_powering_up++; |
| if (IS_NODE_REBOOT_ISSUED(ns)) |
| ps->nodes_reboot_issued++; |
| if (IS_NODE_REBOOT_REQUESTED(ns)) |
| ps->nodes_reboot_requested++; |
| if (IS_NODE_RES(ns)) |
| ps->nodes_resv++; |
| } |
| |
| static int _get_part_statistics(void *x, void *arg) |
| { |
| part_record_t *part_ptr = x; |
| partitions_stats_t *ps = ((foreach_part_gen_stats_t *) arg)->ps; |
| nodes_stats_t *ns = ((foreach_part_gen_stats_t *) arg)->ns; |
| jobs_stats_t *js = ((foreach_part_gen_stats_t *) arg)->js; |
| partition_stats_t *s = xmalloc(sizeof(*s)); |
| |
| s->total_cpus = part_ptr->total_cpus; |
| s->total_nodes = part_ptr->total_nodes; |
| s->name = xstrdup(part_ptr->name); |
| |
| for (int i = 0; i < ns->node_stats_count; i++) { |
| if (!bit_test(part_ptr->node_bitmap, i)) |
| continue; |
| _statistics_part_aggregate_node(s, ns->node_stats_table[i]); |
| } |
| list_for_each_ro(js->jobs, _statistics_part_aggregate_job, s); |
| |
| list_append(ps->parts, s); |
| |
| return SLURM_SUCCESS; |
| } |
| |
| static int _fill_jobs_statistics(void *x, void *arg) |
| { |
| job_record_t *j = x; |
| foreach_fill_jobs_args_t *args = arg; |
| jobs_stats_t *js = args->js; |
| int gpu_tres_pos = args->gpu_tres_pos; |
| job_stats_t *new = xmalloc(sizeof(*new)); |
| |
| new->job_array_cnt = (j->array_recs && j->array_recs->task_cnt) ? |
| j->array_recs->task_cnt : |
| 1; |
| |
| if (IS_JOB_BOOT_FAIL(j)) |
| js->bootfail++; |
| if (IS_JOB_CANCELLED(j)) |
| js->cancelled++; |
| if (IS_JOB_COMPLETE(j)) |
| js->completed++; |
| if (IS_JOB_DEADLINE(j)) |
| js->deadline++; |
| if (IS_JOB_FAILED(j)) |
| js->failed++; |
| if (IS_JOB_NODE_FAILED(j)) |
| js->node_failed++; |
| if (IS_JOB_OOM(j)) |
| js->oom++; |
| if (IS_JOB_PENDING(j)) { |
| if ((j->state_reason == WAIT_HELD) || |
| (j->state_reason == WAIT_HELD_USER)) { |
| js->hold += new->job_array_cnt; |
| } |
| js->pending += new->job_array_cnt; |
| } |
| if (IS_JOB_PREEMPTED(j)) |
| js->preempted++; |
| if (IS_JOB_RUNNING(j)) |
| js->running++; |
| if (IS_JOB_SUSPENDED(j)) |
| js->suspended++; |
| if (IS_JOB_TIMEOUT(j)) |
| js->timeout++; |
| |
| /* Derived job states */ |
| if (IS_JOB_COMPLETING(j)) |
| js->completing++; |
| if (IS_JOB_CONFIGURING(j)) |
| js->configuring++; |
| if (IS_JOB_EXPEDITING(j)) |
| js->expediting++; |
| if (IS_JOB_FED_REQUEUED(j)) |
| js->fed_requeued++; |
| if (IS_JOB_FINISHED(j)) |
| js->finished++; |
| if (IS_JOB_POWER_UP_NODE(j)) |
| js->powerup_node++; |
| if (IS_JOB_REQUEUED(j)) |
| js->requeued++; |
| if (IS_JOB_RESIZING(j)) |
| js->resizing++; |
| if (IS_JOB_REVOKED(j)) |
| js->revoked++; |
| if (IS_JOB_SIGNALING(j)) |
| js->signaling++; |
| if (IS_JOB_STAGE_OUT(j)) |
| js->stageout++; |
| if (IS_JOB_STARTED(j)) |
| js->started++; |
| |
| /* Store individual records */ |
| new->job_id = j->job_id; |
| new->partition = |
| xstrdup(j->part_ptr ? j->part_ptr->name : j->partition); |
| if (!j->user_name) |
| new->user_name = user_from_job(j); |
| else |
| new->user_name = xstrdup(j->user_name); |
| new->account = xstrdup(j->account); |
| |
| if (IS_JOB_RUNNING(j) || IS_JOB_SUSPENDED(j)) { |
| new->cpus_alloc = j->total_cpus; |
| new->memory_alloc = |
| (j->tres_alloc_cnt ? j->tres_alloc_cnt[TRES_ARRAY_MEM] : |
| 0); |
| if (j->node_bitmap) |
| new->node_bitmap = bit_copy(j->node_bitmap); |
| |
| if ((gpu_tres_pos >= 0) && j->tres_alloc_cnt) |
| new->gpus_alloc = j->tres_alloc_cnt[gpu_tres_pos]; |
| |
| js->cpus_alloc += new->cpus_alloc; |
| js->memory_alloc += new->memory_alloc; |
| js->gpus_alloc += new->gpus_alloc; |
| /* |
| * Several jobs can share the same nodes, so update the |
| * node_bitmap of all-jobs-stats adding the bits of this job. |
| */ |
| if (j->node_bitmap) { |
| if (!js->node_bitmap) |
| js->node_bitmap = bit_copy(j->node_bitmap); |
| else |
| bit_or(js->node_bitmap, j->node_bitmap); |
| |
| /* Update the metric with the new count, O(1) cost. */ |
| js->nodes_alloc = bit_set_count(js->node_bitmap); |
| } |
| } |
| |
| /* |
| * Custom Slinky metrics for autoscaling |
| * Max of the min_nodes required of all pending jobs in that partition |
| * Max of the max_nodes required of all pending jobs in that partition |
| */ |
| if (IS_JOB_PENDING(j) && j->details) { |
| new->min_nodes = j->details->min_nodes; |
| new->max_nodes = MAX(j->details->max_nodes, new->min_nodes); |
| } |
| new->job_state = j->job_state; |
| new->state_reason = j->state_reason; |
| list_append(js->jobs, new); |
| |
| js->job_cnt += new->job_array_cnt; |
| |
| return SLURM_SUCCESS; |
| } |
| |
| static void _aggregate_job_to_jobs(jobs_stats_t *s, job_stats_t *j) |
| { |
| if (IS_JOB_BOOT_FAIL(j)) |
| s->bootfail++; |
| if (IS_JOB_CANCELLED(j)) |
| s->cancelled++; |
| if (IS_JOB_COMPLETE(j)) |
| s->completed++; |
| if (IS_JOB_DEADLINE(j)) |
| s->deadline++; |
| if (IS_JOB_FAILED(j)) |
| s->failed++; |
| if (IS_JOB_NODE_FAILED(j)) |
| s->node_failed++; |
| if (IS_JOB_OOM(j)) |
| s->oom++; |
| if (IS_JOB_PENDING(j)) { |
| if ((j->state_reason == WAIT_HELD) || |
| (j->state_reason == WAIT_HELD_USER)) { |
| s->hold += j->job_array_cnt; |
| } |
| s->pending += j->job_array_cnt; |
| } |
| if (IS_JOB_PREEMPTED(j)) |
| s->preempted++; |
| if (IS_JOB_RUNNING(j)) |
| s->running++; |
| if (IS_JOB_SUSPENDED(j)) |
| s->suspended++; |
| if (IS_JOB_TIMEOUT(j)) |
| s->timeout++; |
| |
| /* Derived job states */ |
| if (IS_JOB_COMPLETING(j)) |
| s->completing++; |
| if (IS_JOB_CONFIGURING(j)) |
| s->configuring++; |
| if (IS_JOB_EXPEDITING(j)) |
| s->expediting++; |
| if (IS_JOB_FED_REQUEUED(j)) |
| s->fed_requeued++; |
| if (IS_JOB_FINISHED(j)) |
| s->finished++; |
| if (IS_JOB_POWER_UP_NODE(j)) |
| s->powerup_node++; |
| if (IS_JOB_REQUEUED(j)) |
| s->requeued++; |
| if (IS_JOB_RESIZING(j)) |
| s->resizing++; |
| if (IS_JOB_REVOKED(j)) |
| s->revoked++; |
| if (IS_JOB_SIGNALING(j)) |
| s->signaling++; |
| if (IS_JOB_STAGE_OUT(j)) |
| s->stageout++; |
| if (IS_JOB_STARTED(j)) |
| s->started++; |
| |
| if (IS_JOB_RUNNING(j) || IS_JOB_SUSPENDED(j)) { |
| s->cpus_alloc += j->cpus_alloc; |
| s->gpus_alloc += j->gpus_alloc; |
| s->memory_alloc += j->memory_alloc; |
| /* |
| * Several jobs can share the same nodes, so update the |
| * node_bitmap of all-jobs-stats adding the bits of this job. |
| */ |
| if (j->node_bitmap) { |
| if (!s->node_bitmap) |
| s->node_bitmap = bit_copy(j->node_bitmap); |
| else |
| bit_or(s->node_bitmap, j->node_bitmap); |
| |
| /* Update the metric with the new count, O(1) cost. */ |
| s->nodes_alloc = bit_set_count(s->node_bitmap); |
| } |
| } |
| |
| s->job_cnt++; |
| } |
| |
| static int _get_users_accts(void *x, void *args) |
| { |
| users_accts_stats_t *s = args; |
| job_stats_t *j = x; |
| ua_stats_t *us, *as; |
| |
| us = list_find_first(s->users, _find_ua_by_name, j->user_name); |
| as = list_find_first(s->accounts, _find_ua_by_name, j->account); |
| |
| if (!us) { |
| us = xmalloc(sizeof(*us)); |
| us->s = xmalloc(sizeof(*us->s)); |
| us->name = xstrdup(j->user_name); |
| list_append(s->users, us); |
| } |
| if (!as) { |
| as = xmalloc(sizeof(*as)); |
| as->s = xmalloc(sizeof(*as->s)); |
| as->name = xstrdup(j->account); |
| list_append(s->accounts, as); |
| } |
| |
| _aggregate_job_to_jobs(us->s, j); |
| _aggregate_job_to_jobs(as->s, j); |
| |
| return SLURM_SUCCESS; |
| } |
| |
| extern jobs_stats_t *statistics_get_jobs(bool lock) |
| { |
| slurmctld_lock_t job_read_lock = { |
| .conf = READ_LOCK, |
| .job = READ_LOCK, |
| .part = READ_LOCK, |
| .fed = READ_LOCK, |
| }; |
| jobs_stats_t *s = xmalloc(sizeof(*s)); |
| slurmdb_tres_rec_t tres_rec = { .type = "gres", .name = "gpu" }; |
| |
| /* Avoid looking for tres pos at each iteration, store in args. */ |
| foreach_fill_jobs_args_t args = { |
| .gpu_tres_pos = assoc_mgr_find_tres_pos(&tres_rec, false), |
| .js = s, |
| }; |
| |
| s->jobs = list_create((ListDelF) _free_job_stats); |
| |
| if (lock) |
| lock_slurmctld(job_read_lock); |
| |
| list_for_each_ro(job_list, _fill_jobs_statistics, &args); |
| |
| if (lock) |
| unlock_slurmctld(job_read_lock); |
| |
| return s; |
| } |
| |
| extern nodes_stats_t *statistics_get_nodes(bool lock) |
| { |
| slurmctld_lock_t node_write_lock = { |
| .conf = READ_LOCK, |
| .node = READ_LOCK, |
| .part = READ_LOCK, |
| .select_node = WRITE_LOCK, /*select_g_select_nodeinfo_set_all*/ |
| }; |
| slurmdb_tres_rec_t tres_rec = { .type = "gres", .name = "gpu" }; |
| int gpu_tres_pos = assoc_mgr_find_tres_pos(&tres_rec, false); |
| uint32_t gpu_plugin_id = gres_get_gpu_plugin_id(); |
| node_record_t *node_ptr; |
| nodes_stats_t *s = xmalloc(sizeof(*s)); |
| |
| if (lock) |
| lock_slurmctld(node_write_lock); |
| |
| select_g_select_nodeinfo_set_all(); |
| |
| s->node_stats_table = |
| xcalloc(node_record_count, sizeof(*s->node_stats_table)); |
| s->node_stats_count = node_record_count; |
| |
| for (int i = 0; (node_ptr = next_node(&i)); i++) { |
| uint16_t idle_cpus = 0; |
| node_stats_t *n; |
| node_select_stats_t *select_stats = node_select_stats_array[i]; |
| |
| n = xmalloc(sizeof(*n)); |
| |
| idle_cpus = node_ptr->cpus_efctv - select_stats->alloc_cpus; |
| n->name = xstrdup(node_ptr->name); |
| n->cpus_alloc = select_stats->alloc_cpus; |
| n->cpus_efctv = node_ptr->cpus_efctv; |
| n->cpus_idle = idle_cpus; |
| n->cpus_total = node_ptr->cpus; |
| n->mem_alloc = select_stats->alloc_memory; |
| n->mem_avail = node_ptr->real_memory - node_ptr->mem_spec_limit; |
| n->mem_free = |
| (node_ptr->free_mem == NO_VAL64 ? 0 : |
| node_ptr->free_mem); |
| n->mem_total = node_ptr->real_memory; |
| n->node_state = node_ptr->node_state; |
| |
| /* GPU totals from TRES */ |
| if ((gpu_tres_pos >= 0) && node_ptr->tres_cnt) |
| n->gpus_total = node_ptr->tres_cnt[gpu_tres_pos]; |
| |
| /* GPU allocated from GRES node state */ |
| if (node_ptr->gres_list) { |
| gres_state_t *gres_state_node; |
| gres_state_node = |
| list_find_first(node_ptr->gres_list, |
| gres_find_id, &gpu_plugin_id); |
| if (gres_state_node) { |
| gres_node_state_t *gres_ns = |
| gres_state_node->gres_data; |
| n->gpus_alloc = gres_ns->gres_cnt_alloc; |
| } |
| } |
| |
| s->node_stats_table[i] = n; |
| |
| /* |
| * Base states are unique but can be combined with any other |
| * derived node state |
| */ |
| |
| /* Base states */ |
| if (IS_NODE_FUTURE(node_ptr)) |
| s->future++; |
| else if (IS_NODE_DOWN(node_ptr)) |
| s->down++; |
| else if ((idle_cpus && (idle_cpus < node_ptr->cpus_efctv)) || |
| (select_stats->alloc_tres_fmt_str && |
| (idle_cpus == node_ptr->cpus_efctv))) { |
| n->node_state &= NODE_STATE_FLAGS; |
| n->node_state |= NODE_STATE_MIXED; |
| s->mixed++; |
| /* |
| * The MIXED state is not set by the controller, it must |
| * be inferred - e.g. see also _set_node_mixed() |
| */ |
| } else if (IS_NODE_ALLOCATED(node_ptr)) |
| s->alloc++; |
| else if (IS_NODE_IDLE(node_ptr)) |
| s->idle++; |
| else if (IS_NODE_UNKNOWN(node_ptr)) |
| s->unknown++; |
| |
| /* Derived node states */ |
| if (IS_NODE_COMPLETING(node_ptr)) |
| s->cg++; |
| if (IS_NODE_BLOCKED(node_ptr)) |
| s->blocked++; |
| if (IS_NODE_CLOUD(node_ptr)) |
| s->cloud++; |
| |
| if (IS_NODE_DRAINING(node_ptr)) |
| s->draining++; |
| else if (IS_NODE_DRAIN(node_ptr)) |
| s->drain++; |
| |
| if (IS_NODE_DRAINED(node_ptr)) |
| s->drained++; |
| if (IS_NODE_DYNAMIC_FUTURE(node_ptr)) |
| s->dyn_future++; |
| if (IS_NODE_DYNAMIC_NORM(node_ptr)) |
| s->dyn_normal++; |
| if (IS_NODE_EXTERNAL(node_ptr)) |
| s->external++; |
| if (IS_NODE_FAIL(node_ptr)) |
| s->fail++; |
| if (IS_NODE_INVALID_REG(node_ptr)) |
| s->invalid_reg++; |
| if (IS_NODE_MAINT(node_ptr)) |
| s->maint++; |
| if (IS_NODE_NO_RESPOND(node_ptr)) |
| s->no_resp++; |
| if (IS_NODE_PLANNED(node_ptr)) |
| s->planned++; |
| if (IS_NODE_POWER_DOWN(node_ptr)) |
| s->power_down++; |
| if (IS_NODE_POWER_UP(node_ptr)) |
| s->power_up++; |
| if (IS_NODE_POWERED_DOWN(node_ptr)) |
| s->powered_down++; |
| if (IS_NODE_POWERING_DOWN(node_ptr)) |
| s->powering_down++; |
| if (IS_NODE_POWERING_UP(node_ptr)) |
| s->powering_up++; |
| if (IS_NODE_REBOOT_ISSUED(node_ptr)) |
| s->reboot_issued++; |
| if (IS_NODE_REBOOT_REQUESTED(node_ptr)) |
| s->reboot_requested++; |
| if (IS_NODE_RES(node_ptr)) |
| s->resv++; |
| } |
| |
| if (lock) |
| unlock_slurmctld(node_write_lock); |
| |
| return s; |
| } |
| |
| extern partitions_stats_t *statistics_get_parts(nodes_stats_t *ns, |
| jobs_stats_t *js, bool lock) |
| { |
| slurmctld_lock_t part_read_lock = { |
| .conf = READ_LOCK, |
| .part = READ_LOCK, |
| }; |
| foreach_part_gen_stats_t p = { 0 }; |
| partitions_stats_t *ps = xmalloc(sizeof(*ps)); |
| |
| p.ps = ps; |
| p.ns = ns; |
| p.js = js; |
| |
| ps->parts = list_create((ListDelF) _free_part_stats); |
| |
| if (lock) |
| lock_slurmctld(part_read_lock); |
| |
| list_for_each_ro(part_list, _get_part_statistics, &p); |
| |
| if (lock) |
| unlock_slurmctld(part_read_lock); |
| |
| return ps; |
| } |
| |
| extern scheduling_stats_t *statistics_get_sched(void) |
| { |
| scheduling_stats_t *s = xmalloc(sizeof(*s)); |
| |
| s->agent_queue_size = retry_list_size(); |
| s->agent_count = get_agent_count(); |
| s->agent_thread_count = get_agent_thread_count(); |
| s->diag_stats = xmalloc(sizeof(*(s->diag_stats))); |
| memcpy(s->diag_stats, &slurmctld_diag_stats, |
| sizeof(slurmctld_diag_stats)); |
| |
| /* extended diag stats */ |
| if (s->diag_stats->schedule_cycle_counter > 0) { |
| s->sched_mean_cycle = s->diag_stats->schedule_cycle_sum / |
| s->diag_stats->schedule_cycle_counter; |
| s->sched_mean_depth_cycle = |
| s->diag_stats->schedule_cycle_depth / |
| s->diag_stats->schedule_cycle_counter; |
| } |
| if (s->diag_stats->bf_cycle_counter > 0) { |
| s->bf_depth_mean = s->diag_stats->bf_depth_sum / |
| s->diag_stats->bf_cycle_counter; |
| s->bf_try_depth_mean = s->diag_stats->bf_depth_try_sum / |
| s->diag_stats->bf_cycle_counter; |
| s->bf_queue_len_mean = s->diag_stats->bf_queue_len_sum / |
| s->diag_stats->bf_cycle_counter; |
| s->bf_mean_table_sz = s->diag_stats->bf_table_size_sum / |
| s->diag_stats->bf_cycle_counter; |
| s->bf_mean_cycle = s->diag_stats->bf_cycle_sum / |
| s->diag_stats->bf_cycle_counter; |
| } |
| s->last_proc_req_start = last_proc_req_start; |
| if (acct_storage_g_get_data(acct_db_conn, ACCT_STORAGE_INFO_AGENT_COUNT, |
| &s->slurmdbd_queue_size) != SLURM_SUCCESS) |
| s->slurmdbd_queue_size = 0; |
| s->time = time(NULL); |
| slurm_mutex_lock(&slurmctld_config.thread_count_lock); |
| s->server_thread_count = slurmctld_config.server_thread_count; |
| slurm_mutex_unlock(&slurmctld_config.thread_count_lock); |
| |
| return s; |
| } |
| |
| extern users_accts_stats_t *statistics_get_users_accounts(jobs_stats_t *js) |
| { |
| users_accts_stats_t *ua = xmalloc(sizeof(*ua)); |
| |
| ua->users = list_create((ListDelF) _free_ua_stats); |
| ua->accounts = list_create((ListDelF) _free_ua_stats); |
| |
| list_for_each(js->jobs, _get_users_accts, ua); |
| |
| return ua; |
| } |
| |
| extern void statistics_free_jobs(jobs_stats_t *s) |
| { |
| FREE_NULL_BITMAP(s->node_bitmap); |
| FREE_NULL_LIST(s->jobs); |
| xfree(s); |
| } |
| |
| extern void statistics_free_parts(partitions_stats_t *s) |
| { |
| FREE_NULL_LIST(s->parts); |
| xfree(s); |
| } |
| |
| extern void statistics_free_nodes(nodes_stats_t *s) |
| { |
| for (int i = 0; i < s->node_stats_count; i++) { |
| if (!s->node_stats_table[i]) |
| continue; |
| xfree(s->node_stats_table[i]->name); |
| xfree(s->node_stats_table[i]); |
| } |
| xfree(s->node_stats_table); |
| xfree(s); |
| } |
| |
| extern void statistics_free_sched(scheduling_stats_t *s) |
| { |
| xfree(s->diag_stats); |
| xfree(s); |
| } |
| |
| extern void statistics_free_users_accounts(users_accts_stats_t *s) |
| { |
| FREE_NULL_LIST(s->users); |
| FREE_NULL_LIST(s->accounts); |
| xfree(s); |
| } |
