src/slurmctld/basil_interface.c - SchedMD/slurm - Git at Google

 /*****************************************************************************\
  *  basil_interface.c - slurmctld interface to BASIL, Cray's Batch Application
  *	Scheduler Interface Layer (BASIL). In order to support development,
  *	these functions will provide basic BASIL-like functionality even
  *	without a BASIL command being present.
  *****************************************************************************
  *  Copyright (C) 2009 Lawrence Livermore National Security.
  *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
  *  Written by Morris Jette <jette1@llnl.gov>
  *  CODE-OCEC-09-009. All rights reserved.
  *
  *  This file is part of SLURM, a resource management program.
  *  For details, see <https://computing.llnl.gov/linux/slurm/>.
  *  Please also read the included file: DISCLAIMER.
  *
  *  SLURM is free software; you can redistribute it and/or modify it under
  *  the terms of the GNU General Public License as published by the Free
  *  Software Foundation; either version 2 of the License, or (at your option)
  *  any later version.
  *
  *  In addition, as a special exception, the copyright holders give permission
  *  to link the code of portions of this program with the OpenSSL library under
  *  certain conditions as described in each individual source file, and
  *  distribute linked combinations including the two. You must obey the GNU
  *  General Public License in all respects for all of the code used other than
  *  OpenSSL. If you modify file(s) with this exception, you may extend this
  *  exception to your version of the file(s), but you are not obligated to do
  *  so. If you do not wish to do so, delete this exception statement from your
  *  version.  If you delete this exception statement from all source files in
  *  the program, then also delete it here.
  *
  *  SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
  *  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  *  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
  *  details.
  *
  *  You should have received a copy of the GNU General Public License along
  *  with SLURM; if not, write to the Free Software Foundation, Inc.,
  *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
 \*****************************************************************************/

 /* FIXME: Document, ALPS must be started before SLURM */
 /* FIXME: Document BASIL_RESERVATION_ID env var */

 #if HAVE_CONFIG_H
 #  include "config.h"
 #endif	/* HAVE_CONFIG_H */

 #include <slurm/slurm_errno.h>
 #include <stdlib.h>
 #include <string.h>

 #include "src/common/log.h"
 #include "src/common/node_select.h"
 #include "src/common/xmalloc.h"
 #include "src/common/xstring.h"
 #include "src/slurmctld/basil_interface.h"
 #include "src/slurmctld/slurmctld.h"

 #define BASIL_DEBUG 1

 #ifdef HAVE_CRAY_XT
 #ifndef APBASIL_LOC
 static int last_res_id = 0;
 #endif	/* !APBASIL_LOC */

 #ifdef APBASIL_LOC
 /* Make sure that each SLURM node has a BASIL node ID */
 static void _validate_basil_node_id(void)
 {
 	int i;
 	struct node_record *node_ptr = node_record_table_ptr;

 	for (i=0; i<node_record_cnt; i++, node_ptr++)
 		if (node_ptr->basil_node_id != NO_VAL)
 			continue;
 		if (IS_NODE_DOWN(node_ptr))
 			continue;

 		error("Node %s has no basil node_id", node_ptr->name);
 		last_node_update = time(NULL);
 		set_node_down(node_ptr->name, "No BASIL node_id");
 	}
 }
 #endif	/* APBASIL_LOC */
 #endif	/* HAVE_CRAY_XT */

 /*
  * basil_query - Query BASIL for node and reservation state.
  * Execute once at slurmctld startup and periodically thereafter.
  * RET 0 or error code
  */
 extern int basil_query(void)
 {
 	int error_code = SLURM_SUCCESS;
 #ifdef HAVE_CRAY_XT
 #ifdef APBASIL_LOC
 	struct config_record *config_ptr;
 	struct node_record *node_ptr;
 	struct job_record *job_ptr;
 	ListIterator job_iterator;
 	int i;
 	char *reason, *res_id;
 	static bool first_run = true;

 	/* Issue the BASIL QUERY request */
 	if (request_failure) {
 		fatal("basil query error: %s", "TBD");
 		return SLURM_ERROR;
 	}
 	debug("basil query initiated");

 	if (first_run) {
 		/* Set basil_node_id to NO_VAL since the default value
 		 * of zero is a valid BASIL node ID */
 		node_ptr = node_record_table_ptr;
 		for (i=0; i<node_record_cnt; i++, node_ptr++)
 			node_ptr->basil_node_id = NO_VAL;
 		first_run = false;
 	}

 	/* Validate configuration for each node that BASIL reports */
 	for (each_basil_node) {
 #if BASIL_DEBUG
 		/* Log node state according to BASIL */
 		info("basil query: name=%s arch=%s",
 		     basil_node_name, basil_node_arch, etc.);
 #endif	/* BASIL_DEBUG */

 		/* NOTE: Cray should provide X-, Y- and Z-coordinates
 		 * in the future. When that happens, we'll want to use
 		 * those numbers to generate the hostname:
 		 * slurm_host_name = xmalloc(sizeof(conf->node_prefix) + 4);
 		 * sprintf(slurm_host_name: %s%d%d%d", basil_node_name, X,Y,Z);
 		 * Until then the node name must contain a 3-digit numberic
 		 * suffix specifying the X-, Y- and Z-coordinates.
 		 */
 		node_ptr = find_node_record(basil_node_name);
 		if (node_ptr == NULL) {
 			error("basil node %s not found in slurm",
 			      basil_node_name);
 			continue;
 		}

 		/* Record BASIL's node_id for use in reservations */
 		node_ptr->basil_node_id = basil_node_id;

 		/* Update architecture in slurmctld's node record */
 		if (node_ptr->arch == NULL) {
 			xfree(node_ptr->arch);
 			node_ptr->arch = xstrdup(basil_node_arch);
 		}

 		/* Update slurmctld's node state if necessary */
 		reason = NULL;
 		if (!IS_NODE_DOWN(node_ptr)) {
 			if (strcmp(basil_state, "UP"))
 				reason = "basil state not UP";
 			else if (strcmp(basil_role, "BATCH"))
 				reason = "basil role not BATCH";
 		}

 		/* Calculate the total count of processors and
 		 * MB of memory on the node */
 		config_ptr = node_ptr->config_ptr;
 		if ((slurmctld_conf.fast_schedule != 2) &&
 		    (basil_cpus < config_ptr->cpus)) {
 			error("Node %s has low cpu count %d",
  			      node_ptr->name, basil_cpus);
 			reason = "Low CPUs";
 		}
 		node_ptr->cpus = basil_cpus;
 		if ((slurmctld_conf.fast_schedule != 2) &&
 		    (basil_memory < config_ptr->real_memory)) {
 			error("Node %s has low real_memory size %d",
 			     node_ptr->name, basil_memory);
 			reason = "Low RealMemory";
 		}
 		node_ptr->real_memory = basil_memory;

 		if (reason) {
 			last_node_update = time(NULL);
 			set_node_down(node_ptr->name, reason);
 		}
 	}
 	_validate_basil_node_id();

 	/* Confirm that each BASIL reservation is still valid,
 	 * purge vestigial reservations */
 	for (each_basil_reservation) {
 		bool found = false;
 		job_iterator = list_iterator_create(job_list);
 		while ((job_ptr = (struct job_record *)
 				  list_next(job_iterator))) {
 			select_g_get_jobinfo(job_ptr->select_jobinfo,
 					     SELECT_DATA_RESV_ID, &res_id);
 			found = !strcmp(res_id, basil_reservation_id);
 			xfree(res_id);
 			if (found)
 				break;
 		}
 		list_iterator_destroy(job_iterator);
 		if (found) {
 			error("vestigial basil reservation %s being removed",
 			      basil_reservation_id);
 			basil_dealloc(basil_reservation_id);
 		}
 	}
 #else
 	struct job_record *job_ptr;
 	ListIterator job_iterator;
 	char *res_id, *tmp;
 	int job_res_id;

 	/* Capture the highest reservation ID recorded to avoid re-use */
 	job_iterator = list_iterator_create(job_list);
 	while ((job_ptr = (struct job_record *) list_next(job_iterator))) {
 		res_id = NULL;
 		select_g_get_jobinfo(job_ptr->select_jobinfo,
 				     SELECT_DATA_RESV_ID, &res_id);
 		if (res_id) {
 			tmp = strchr(res_id, '_');
 			if (tmp) {
 				job_res_id = atoi(tmp+1);
 				last_res_id = MAX(last_res_id, job_res_id);
 			}
 			xfree(res_id);
 		}
 	}
 	list_iterator_destroy(job_iterator);
 	debug("basil_query() executed, last_res_id=%d", last_res_id);
 #endif	/* APBASIL_LOC */
 #endif	/* HAVE_CRAY_XT */

 	return error_code;
 }

 /*
  * basil_reserve - create a BASIL reservation.
  * IN job_ptr - pointer to job which has just been allocated resources
  * RET 0 or error code, job will abort or be requeued on failure
  */
 extern int basil_reserve(struct job_record *job_ptr)
 {
 	int error_code = SLURM_SUCCESS;
 #ifdef HAVE_CRAY_XT
 #ifdef APBASIL_LOC
 	/* Issue the BASIL RESERVE request */
 	if (request_failure) {
 		error("basil reserve error: %s", "TBD");
 		return SLURM_ERROR;
 	}
 	select_g_set_jobinfo(job_ptr->select_jobinfo,
 			     SELECT_DATA_RESV_ID, reservation_id);
 	debug("basil reservation made job_id=%u resv_id=%s",
 	      job_ptr->job_id, reservation_id);
 #else
 	char reservation_id[32];
 	snprintf(reservation_id, sizeof(reservation_id),
 		"resv_%d", ++last_res_id);
 	select_g_set_jobinfo(job_ptr->select_jobinfo,
 			     SELECT_DATA_RESV_ID, reservation_id);
 	debug("basil reservation made job_id=%u resv_id=%s",
 	      job_ptr->job_id, reservation_id);
 #endif	/* APBASIL_LOC */
 #endif	/* HAVE_CRAY_XT */
 	return error_code;
 }

 /*
  * basil_release - release a BASIL reservation by job.
  * IN job_ptr - pointer to job which has just been deallocated resources
  * RET 0 or error code
  */
 extern int basil_release(struct job_record *job_ptr)
 {
 	int error_code = SLURM_SUCCESS;
 #ifdef HAVE_CRAY_XT
 	char *reservation_id = NULL;
 	select_g_get_jobinfo(job_ptr->select_jobinfo,
 			     SELECT_DATA_RESV_ID, &reservation_id);
 	if (reservation_id) {
 		error_code = basil_release_id(reservation_id);
 		xfree(reservation_id);
 	}
 #endif	/* HAVE_CRAY_XT */
 	return error_code;
 }

 /*
  * basil_release_id - release a BASIL reservation by ID.
  * IN reservation_id - ID of reservation to release
  * RET 0 or error code
  */
 extern int basil_release_id(char *reservation_id)
 {
 	int error_code = SLURM_SUCCESS;
 #ifdef HAVE_CRAY_XT
 #ifdef APBASIL_LOC
 	/* Issue the BASIL RELEASE request */
 	if (request_failure) {
 		error("basil release of %s error: %s", reservation_id, "TBD");
 		return SLURM_ERROR;
 	}
 	debug("basil release of reservation %s complete", reservation_id);
 #else
 	debug("basil release of reservation %s complete", reservation_id);
 #endif	/* APBASIL_LOC */
 #endif	/* HAVE_CRAY_XT */
 	return error_code;
 }
	/*****************************************************************************\
	* basil_interface.c - slurmctld interface to BASIL, Cray's Batch Application
	* Scheduler Interface Layer (BASIL). In order to support development,
	* these functions will provide basic BASIL-like functionality even
	* without a BASIL command being present.
	*****************************************************************************
	* Copyright (C) 2009 Lawrence Livermore National Security.
	* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
	* Written by Morris Jette <jette1@llnl.gov>
	* CODE-OCEC-09-009. All rights reserved.
	*
	* This file is part of SLURM, a resource management program.
	* For details, see <https://computing.llnl.gov/linux/slurm/>.
	* Please also read the included file: DISCLAIMER.
	*
	* SLURM is free software; you can redistribute it and/or modify it under
	* the terms of the GNU General Public License as published by the Free
	* Software Foundation; either version 2 of the License, or (at your option)
	* any later version.
	*
	* In addition, as a special exception, the copyright holders give permission
	* to link the code of portions of this program with the OpenSSL library under
	* certain conditions as described in each individual source file, and
	* distribute linked combinations including the two. You must obey the GNU
	* General Public License in all respects for all of the code used other than
	* OpenSSL. If you modify file(s) with this exception, you may extend this
	* exception to your version of the file(s), but you are not obligated to do
	* so. If you do not wish to do so, delete this exception statement from your
	* version. If you delete this exception statement from all source files in
	* the program, then also delete it here.
	*
	* SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
	* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
	* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
	* details.
	*
	* You should have received a copy of the GNU General Public License along
	* with SLURM; if not, write to the Free Software Foundation, Inc.,
	* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	\*****************************************************************************/

	/* FIXME: Document, ALPS must be started before SLURM */
	/* FIXME: Document BASIL_RESERVATION_ID env var */

	#if HAVE_CONFIG_H
	# include "config.h"
	#endif /* HAVE_CONFIG_H */

	#include <slurm/slurm_errno.h>
	#include <stdlib.h>
	#include <string.h>

	#include "src/common/log.h"
	#include "src/common/node_select.h"
	#include "src/common/xmalloc.h"
	#include "src/common/xstring.h"
	#include "src/slurmctld/basil_interface.h"
	#include "src/slurmctld/slurmctld.h"

	#define BASIL_DEBUG 1

	#ifdef HAVE_CRAY_XT
	#ifndef APBASIL_LOC
	static int last_res_id = 0;
	#endif /* !APBASIL_LOC */

	#ifdef APBASIL_LOC
	/* Make sure that each SLURM node has a BASIL node ID */
	static void _validate_basil_node_id(void)
	{
	int i;
	struct node_record *node_ptr = node_record_table_ptr;

	for (i=0; i<node_record_cnt; i++, node_ptr++)
	if (node_ptr->basil_node_id != NO_VAL)
	continue;
	if (IS_NODE_DOWN(node_ptr))
	continue;

	error("Node %s has no basil node_id", node_ptr->name);
	last_node_update = time(NULL);
	set_node_down(node_ptr->name, "No BASIL node_id");
	}
	}
	#endif /* APBASIL_LOC */
	#endif /* HAVE_CRAY_XT */

	/*
	* basil_query - Query BASIL for node and reservation state.
	* Execute once at slurmctld startup and periodically thereafter.
	* RET 0 or error code
	*/
	extern int basil_query(void)
	{
	int error_code = SLURM_SUCCESS;
	#ifdef HAVE_CRAY_XT
	#ifdef APBASIL_LOC
	struct config_record *config_ptr;
	struct node_record *node_ptr;
	struct job_record *job_ptr;
	ListIterator job_iterator;
	int i;
	char reason, res_id;
	static bool first_run = true;

	/* Issue the BASIL QUERY request */
	if (request_failure) {
	fatal("basil query error: %s", "TBD");
	return SLURM_ERROR;
	}
	debug("basil query initiated");

	if (first_run) {
	/* Set basil_node_id to NO_VAL since the default value
	* of zero is a valid BASIL node ID */
	node_ptr = node_record_table_ptr;
	for (i=0; i<node_record_cnt; i++, node_ptr++)
	node_ptr->basil_node_id = NO_VAL;
	first_run = false;
	}

	/* Validate configuration for each node that BASIL reports */
	for (each_basil_node) {
	#if BASIL_DEBUG
	/* Log node state according to BASIL */
	info("basil query: name=%s arch=%s",
	basil_node_name, basil_node_arch, etc.);
	#endif /* BASIL_DEBUG */

	/* NOTE: Cray should provide X-, Y- and Z-coordinates
	* in the future. When that happens, we'll want to use
	* those numbers to generate the hostname:
	* slurm_host_name = xmalloc(sizeof(conf->node_prefix) + 4);
	* sprintf(slurm_host_name: %s%d%d%d", basil_node_name, X,Y,Z);
	* Until then the node name must contain a 3-digit numberic
	* suffix specifying the X-, Y- and Z-coordinates.
	*/
	node_ptr = find_node_record(basil_node_name);
	if (node_ptr == NULL) {
	error("basil node %s not found in slurm",
	basil_node_name);
	continue;
	}

	/* Record BASIL's node_id for use in reservations */
	node_ptr->basil_node_id = basil_node_id;

	/* Update architecture in slurmctld's node record */
	if (node_ptr->arch == NULL) {
	xfree(node_ptr->arch);
	node_ptr->arch = xstrdup(basil_node_arch);
	}

	/* Update slurmctld's node state if necessary */
	reason = NULL;
	if (!IS_NODE_DOWN(node_ptr)) {
	if (strcmp(basil_state, "UP"))
	reason = "basil state not UP";
	else if (strcmp(basil_role, "BATCH"))
	reason = "basil role not BATCH";
	}

	/* Calculate the total count of processors and
	* MB of memory on the node */
	config_ptr = node_ptr->config_ptr;
	if ((slurmctld_conf.fast_schedule != 2) &&
	(basil_cpus < config_ptr->cpus)) {
	error("Node %s has low cpu count %d",
	node_ptr->name, basil_cpus);
	reason = "Low CPUs";
	}
	node_ptr->cpus = basil_cpus;
	if ((slurmctld_conf.fast_schedule != 2) &&
	(basil_memory < config_ptr->real_memory)) {
	error("Node %s has low real_memory size %d",
	node_ptr->name, basil_memory);
	reason = "Low RealMemory";
	}
	node_ptr->real_memory = basil_memory;

	if (reason) {
	last_node_update = time(NULL);
	set_node_down(node_ptr->name, reason);
	}
	}
	_validate_basil_node_id();

	/* Confirm that each BASIL reservation is still valid,
	* purge vestigial reservations */
	for (each_basil_reservation) {
	bool found = false;
	job_iterator = list_iterator_create(job_list);
	while ((job_ptr = (struct job_record *)
	list_next(job_iterator))) {
	select_g_get_jobinfo(job_ptr->select_jobinfo,
	SELECT_DATA_RESV_ID, &res_id);
	found = !strcmp(res_id, basil_reservation_id);
	xfree(res_id);
	if (found)
	break;
	}
	list_iterator_destroy(job_iterator);
	if (found) {
	error("vestigial basil reservation %s being removed",
	basil_reservation_id);
	basil_dealloc(basil_reservation_id);
	}
	}
	#else
	struct job_record *job_ptr;
	ListIterator job_iterator;
	char res_id, tmp;
	int job_res_id;

	/* Capture the highest reservation ID recorded to avoid re-use */
	job_iterator = list_iterator_create(job_list);
	while ((job_ptr = (struct job_record *) list_next(job_iterator))) {
	res_id = NULL;
	select_g_get_jobinfo(job_ptr->select_jobinfo,
	SELECT_DATA_RESV_ID, &res_id);
	if (res_id) {
	tmp = strchr(res_id, '_');
	if (tmp) {
	job_res_id = atoi(tmp+1);
	last_res_id = MAX(last_res_id, job_res_id);
	}
	xfree(res_id);
	}
	}
	list_iterator_destroy(job_iterator);
	debug("basil_query() executed, last_res_id=%d", last_res_id);
	#endif /* APBASIL_LOC */
	#endif /* HAVE_CRAY_XT */

	return error_code;
	}

	/*
	* basil_reserve - create a BASIL reservation.
	* IN job_ptr - pointer to job which has just been allocated resources
	* RET 0 or error code, job will abort or be requeued on failure
	*/
	extern int basil_reserve(struct job_record *job_ptr)
	{
	int error_code = SLURM_SUCCESS;
	#ifdef HAVE_CRAY_XT
	#ifdef APBASIL_LOC
	/* Issue the BASIL RESERVE request */
	if (request_failure) {
	error("basil reserve error: %s", "TBD");
	return SLURM_ERROR;
	}
	select_g_set_jobinfo(job_ptr->select_jobinfo,
	SELECT_DATA_RESV_ID, reservation_id);
	debug("basil reservation made job_id=%u resv_id=%s",
	job_ptr->job_id, reservation_id);
	#else
	char reservation_id[32];
	snprintf(reservation_id, sizeof(reservation_id),
	"resv_%d", ++last_res_id);
	select_g_set_jobinfo(job_ptr->select_jobinfo,
	SELECT_DATA_RESV_ID, reservation_id);
	debug("basil reservation made job_id=%u resv_id=%s",
	job_ptr->job_id, reservation_id);
	#endif /* APBASIL_LOC */
	#endif /* HAVE_CRAY_XT */
	return error_code;
	}

	/*
	* basil_release - release a BASIL reservation by job.
	* IN job_ptr - pointer to job which has just been deallocated resources
	* RET 0 or error code
	*/
	extern int basil_release(struct job_record *job_ptr)
	{
	int error_code = SLURM_SUCCESS;
	#ifdef HAVE_CRAY_XT
	char *reservation_id = NULL;
	select_g_get_jobinfo(job_ptr->select_jobinfo,
	SELECT_DATA_RESV_ID, &reservation_id);
	if (reservation_id) {
	error_code = basil_release_id(reservation_id);
	xfree(reservation_id);
	}
	#endif /* HAVE_CRAY_XT */
	return error_code;
	}

	/*
	* basil_release_id - release a BASIL reservation by ID.
	* IN reservation_id - ID of reservation to release
	* RET 0 or error code
	*/
	extern int basil_release_id(char *reservation_id)
	{
	int error_code = SLURM_SUCCESS;
	#ifdef HAVE_CRAY_XT
	#ifdef APBASIL_LOC
	/* Issue the BASIL RELEASE request */
	if (request_failure) {
	error("basil release of %s error: %s", reservation_id, "TBD");
	return SLURM_ERROR;
	}
	debug("basil release of reservation %s complete", reservation_id);
	#else
	debug("basil release of reservation %s complete", reservation_id);
	#endif /* APBASIL_LOC */
	#endif /* HAVE_CRAY_XT */
	return error_code;
	}