srp_daemon/srp_daemon.c

/*
 * srp_daemon - discover SRP targets over IB
 * Copyright (c) 2005 Topspin Communications.  All rights reserved.
 * Copyright (c) 2006 Cisco Systems, Inc.  All rights reserved.
 * Copyright (c) 2006 Mellanox Technologies Ltd.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 * $Author: ishai Rabinovitz [ishai@mellanox.co.il]$
 * Based on Roland Dreier's initial code [rdreier@cisco.com]
 *
 */

#define _GNU_SOURCE

#include <assert.h>
#include <stdarg.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <ctype.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <fcntl.h>
#include <linux/types.h>
#include <endian.h>
#include <errno.h>
#include <getopt.h>
#include <dirent.h>
#include <pthread.h>
#include <string.h>
#include <signal.h>
#include <sys/syslog.h>
#include <infiniband/umad.h>
#include <infiniband/umad_types.h>
#include <infiniband/umad_sa.h>
#include "srp_ib_types.h"

#include "srp_daemon.h"

#define IBDEV_STR_SIZE 16
#define IBPORT_STR_SIZE 16
#define IGNORE(value) do { if (value) { } } while (0)
#define max_t(type, x, y) ({                    \
	type __max1 = (x);	\
	type __max2 = (y);	\
	__max1 > __max2 ? __max1: __max2; })

#define get_data_ptr(mad) ((void *) ((mad).hdr.data))

enum log_dest { log_to_syslog, log_to_stderr };

static int get_lid(struct umad_resources *umad_res, union umad_gid *gid,
		   uint16_t *lid);

static const int   node_table_response_size = 1 << 18;
static const char *sysfs_path = "/sys";
static enum log_dest s_log_dest = log_to_syslog;
static int wakeup_pipe[2] = { -1, -1 };


void wake_up_main_loop(char ch)
{
	int res;

	assert(wakeup_pipe[1] >= 0);
	res = write(wakeup_pipe[1], &ch, 1);
	IGNORE(res);
}

static void signal_handler(int signo)
{
	wake_up_main_loop(signo);
}

/*
 * Return either the received signal (SIGINT, SIGTERM, ...) or 0 if no signal
 * has been received before the timeout has expired.
 */
static int get_received_signal(time_t tv_sec, suseconds_t tv_usec)
{
	int fd, ret, received_signal = 0;
	fd_set rset;
	struct timeval timeout;
	char buf[16];

	fd = wakeup_pipe[0];
	FD_ZERO(&rset);
	FD_SET(fd, &rset);
	timeout.tv_sec = tv_sec;
	timeout.tv_usec = tv_usec;
	ret = select(fd + 1, &rset, NULL, NULL, &timeout);
	if (ret < 0)
		assert(errno == EINTR);
	while ((ret = read(fd, buf, sizeof(buf))) > 0)
		received_signal = buf[ret - 1];

	return received_signal;
}

static int check_process_uniqueness(struct config_t *conf)
{
	char path[256];
	int fd;

	snprintf(path, sizeof(path), SRP_DAEMON_LOCK_PREFIX "_%s_%d",
		 conf->dev_name, conf->port_num);

	if ((fd = open(path, O_CREAT|O_RDWR,
		       S_IRUSR|S_IRGRP|S_IROTH|S_IWUSR)) < 0) {
		pr_err("cannot open file \"%s\" (errno: %d).\n", path, errno);
		return -1;
	}

	if (0 != lockf(fd, F_TLOCK, 0)) {
		pr_err("failed to lock %s (errno: %d). possibly another "
		       "srp_daemon is locking it\n", path, errno);
		close(fd);
		fd = -1;
	}

	return fd;
}

static int srpd_sys_read_string(const char *dir_name, const char *file_name,
				char *str, int max_len)
{
	char path[256], *s;
	int fd, r;

	snprintf(path, sizeof(path), "%s/%s", dir_name, file_name);

	if ((fd = open(path, O_RDONLY)) < 0)
		return (errno > 0) ? -errno : errno;

	if ((r = read(fd, str, max_len)) < 0) {
		int e = errno;
		close(fd);
		return (e > 0) ? -e : e;
	}

	str[(r < max_len) ? r : max_len - 1] = 0;

	if ((s = strrchr(str, '\n')))
		*s = 0;

	close(fd);
	return 0;
}

static int srpd_sys_read_gid(const char *dir_name, const char *file_name,
			     uint8_t *gid)
{
	char buf[64], *str, *s;
	__be16 *ugid = (__be16 *)gid;
	int r, i;

	if ((r = srpd_sys_read_string(dir_name, file_name, buf, sizeof(buf))) < 0)
		return r;

	for (s = buf, i = 0 ; i < 8; i++) {
		if (!(str = strsep(&s, ": \t\n")))
			return -EINVAL;
		ugid[i] = htobe16(strtoul(str, NULL, 16) & 0xffff);
	}

	return 0;
}

static int srpd_sys_read_uint64(const char *dir_name, const char *file_name,
				uint64_t *u)
{
	char buf[32];
	int r;

	if ((r = srpd_sys_read_string(dir_name, file_name, buf, sizeof(buf))) < 0)
		return r;

	*u = strtoull(buf, NULL, 0);

	return 0;
}




static void usage(const char *argv0)
{
	fprintf(stderr, "Usage: %s [-vVcaeon] [-d <umad device> | -i <infiniband device> [-p <port_num>]] [-t <timeout (ms)>] [-r <retries>] [-R <rescan time>] [-f <rules file>\n", argv0);
	fprintf(stderr, "-v 			Verbose\n");
	fprintf(stderr, "-V 			debug Verbose\n");
	fprintf(stderr, "-c 			prints connection Commands\n");
	fprintf(stderr, "-a 			show All - prints also targets that are already connected\n");
	fprintf(stderr, "-e 			Executes connection commands\n");
	fprintf(stderr, "-o 			runs only Once and stop\n");
	fprintf(stderr, "-d <umad device>	use umad Device \n");
	fprintf(stderr, "-i <infiniband device>	use InfiniBand device \n");
	fprintf(stderr, "-p <port_num>		use Port num \n");
	fprintf(stderr, "-j <dev>:<port_num>	use the IB dev / port_num combination \n");
	fprintf(stderr, "-R <rescan time>	perform complete Rescan every <rescan time> seconds\n");
	fprintf(stderr, "-T <retry timeout>	Retries to connect to existing target after Timeout of <retry timeout> seconds\n");
	fprintf(stderr, "-l <tl_retry timeout>	Transport retry count before failing IO. should be in range [2..7], (default 2)\n");
	fprintf(stderr, "-f <rules file>	use rules File to set to which target(s) to connect (default: " SRP_DAEMON_CONFIG_FILE ")\n");
	fprintf(stderr, "-t <timeout>		Timeout for mad response in milliseconds\n");
	fprintf(stderr, "-r <retries>		number of send Retries for each mad\n");
	fprintf(stderr, "-n 			New connection command format - use also initiator extension\n");
	fprintf(stderr, "--systemd		Enable systemd integration.\n");
	fprintf(stderr, "\nExample: srp_daemon -e -n -i mthca0 -p 1 -R 60\n");
}

static int
check_equal_uint64(char *dir_name, const char *attr, uint64_t val)
{
	uint64_t attr_value;

	if (srpd_sys_read_uint64(dir_name, attr, &attr_value))
		return 0;

	return attr_value == val;
}

static int
check_equal_uint16(char *dir_name, const char *attr, uint16_t val)
{
	uint64_t attr_value;

	if (srpd_sys_read_uint64(dir_name, attr, &attr_value))
		return 0;

	return val == (attr_value & 0xffff);
}

static int recalc(struct resources *res);

static void pr_cmd(char *target_str, int not_connected)
{
	int ret;

	if (config->cmd)
		printf("%s\n", target_str);

	if (config->execute && not_connected) {
		int fd = open(config->add_target_file, O_WRONLY);
		if (fd < 0) {
			pr_err("unable to open %s, maybe ib_srp is not loaded\n", config->add_target_file);
			return;
		}
		ret = write(fd, target_str, strlen(target_str));
		pr_debug("Adding target returned %d\n", ret);
		close(fd);
	}
}

void pr_debug(const char *fmt, ...)
{
	va_list args;

	if (!config->debug_verbose)
		return;

	va_start(args, fmt);
	vprintf(fmt, args);
	va_end(args);
}

void pr_err(const char *fmt, ...)
{
	va_list args;

	va_start(args, fmt);
	switch (s_log_dest) {
	case log_to_syslog:
		vsyslog(LOG_DAEMON | LOG_ERR, fmt, args);
		break;
	case log_to_stderr:
		vfprintf(stderr, fmt, args);
		break;
	}
	va_end(args);
}

static int check_not_equal_str(const char *dir_name, const char *attr,
			       const char *value)
{
	char attr_value[64];
	int len = strlen(value);

	if (len > sizeof(attr_value)) {
		pr_err("string %s is too long\n", value);
		return 1;
	}

	if (srpd_sys_read_string(dir_name, attr, attr_value,
				 sizeof(attr_value)))
		return 0;
	if (strncmp(attr_value, value, len))
		return 1;

	return 0;
}

static int check_not_equal_int(const char *dir_name, const char *attr,
			       int value)
{
	char attr_value[64];

	if (srpd_sys_read_string(dir_name, attr, attr_value,
				 sizeof(attr_value)))
		return 0;
	if (value != atoi(attr_value))
		return 1;

	return 0;
}

static int is_enabled_by_rules_file(struct target_details *target)
{
	int rule;
	struct config_t *conf = config;

	if (NULL == conf->rules) {
		pr_debug("Allowing SRP target with id_ext %s because not using a rules file\n", target->id_ext);
		return 1;
	}

	rule = -1;
	do {
		rule++;
		if (conf->rules[rule].id_ext[0] != '\0' &&
		    strtoull(target->id_ext, NULL, 16) !=
		    strtoull(conf->rules[rule].id_ext, NULL, 16))
			continue;

		if (conf->rules[rule].ioc_guid[0] != '\0' &&
		    be64toh(target->ioc_prof.guid) !=
		    strtoull(conf->rules[rule].ioc_guid, NULL, 16))
			continue;

		if (conf->rules[rule].dgid[0] != '\0') {
			char tmp = conf->rules[rule].dgid[16];
			conf->rules[rule].dgid[16] = '\0';
			if (strtoull(conf->rules[rule].dgid, NULL, 16) !=
			    target->subnet_prefix) {
				conf->rules[rule].dgid[16] = tmp;
				continue;
			}
			conf->rules[rule].dgid[16] = tmp;
			if (strtoull(&conf->rules[rule].dgid[16], NULL, 16) !=
			    target->h_guid)
				continue;
		}

		if (conf->rules[rule].service_id[0] != '\0' &&
		    strtoull(conf->rules[rule].service_id, NULL, 16) !=
	            target->h_service_id)
			continue;

		if (conf->rules[rule].pkey[0] != '\0' &&
		    (uint16_t)strtoul(conf->rules[rule].pkey, NULL, 16) !=
	            target->pkey)
			continue;

		target->options = conf->rules[rule].options;

		pr_debug("SRP target with id_ext %s %s by rules file\n",
				target->id_ext,
				conf->rules[rule].allow ? "allowed" : "disallowed");
		return conf->rules[rule].allow;

	} while (1);
}


static bool use_imm_data(void)
{
	bool ret = false;
	char flag = 0;
	int cnt;
	int fd = open("/sys/module/ib_srp/parameters/use_imm_data", O_RDONLY);

	if (fd < 0)
		return false;
	cnt = read(fd, &flag, 1);
	if (cnt != 1) {
		close(fd);
		return false;
	}

	if (!strncmp(&flag, "Y", 1))
		ret = true;
	close(fd);
	return ret;
}

static bool imm_data_size_gt_send_size(unsigned int send_size)
{
	bool ret = false;
	unsigned int srp_max_imm_data = 0;
	FILE *fp = fopen("/sys/module/ib_srp/parameters/max_imm_data", "r");
	int cnt;

	if (fp == NULL)
		return ret;

	cnt = fscanf(fp, "%d", &srp_max_imm_data);
	if (cnt <= 0) {
		fclose(fp);
		return ret;
	}

	if (srp_max_imm_data > send_size)
		ret = true;

	fclose(fp);
	return ret;
}

static int add_non_exist_target(struct target_details *target)
{
	char scsi_host_dir[256];
	DIR *dir;
	struct dirent *subdir;
	char *subdir_name_ptr;
	int prefix_len;
	union umad_gid dgid_val;
	char target_config_str[255];
	int len;
	int not_connected = 1;
	unsigned int send_size;

	pr_debug("Found an SRP target with id_ext %s - check if it is already connected\n", target->id_ext);

	strcpy(scsi_host_dir, "/sys/class/scsi_host/");
	dir=opendir(scsi_host_dir);
	if (!dir) {
		perror("opendir - /sys/class/scsi_host/");
		return -1;
	}
	prefix_len = strlen(scsi_host_dir);
	subdir_name_ptr = scsi_host_dir + prefix_len;

	subdir = (void *) 1; /* Dummy value to enter the loop */
	while (subdir) {
	        subdir = readdir(dir);

		if (!subdir)
			continue;

		if (subdir->d_name[0] == '.')
			continue;

		strncpy(subdir_name_ptr, subdir->d_name,
			sizeof(scsi_host_dir) - prefix_len);
		if (!check_equal_uint64(scsi_host_dir, "id_ext",
				        strtoull(target->id_ext, NULL, 16)))
			continue;
		if (!check_equal_uint16(scsi_host_dir, "pkey", target->pkey) &&
		    !config->execute)
			continue;

		if (!check_equal_uint64(scsi_host_dir, "service_id",
					target->h_service_id))
			continue;
		if (!check_equal_uint64(scsi_host_dir, "ioc_guid",
					be64toh(target->ioc_prof.guid)))
			continue;
		if (srpd_sys_read_gid(scsi_host_dir, "orig_dgid",
				      dgid_val.raw)) {
			/*
			 * In case this is an old kernel that does not have
			 * orig_dgid in sysfs, use dgid instead (this is
			 * problematic when there is a dgid redirection
			 * by the CM)
			 */
			if (srpd_sys_read_gid(scsi_host_dir, "dgid",
					      dgid_val.raw))
				continue;
		}
		if (htobe64(target->subnet_prefix) !=
		    dgid_val.global.subnet_prefix)
			continue;
		if (htobe64(target->h_guid) != dgid_val.global.interface_id)
			continue;

		/* If there is no local_ib_device in the scsi host dir (old kernel module), assumes it is equal */
		if (check_not_equal_str(scsi_host_dir, "local_ib_device", config->dev_name))
			continue;

		/* If there is no local_ib_port in the scsi host dir (old kernel module), assumes it is equal */
		if (check_not_equal_int(scsi_host_dir, "local_ib_port", config->port_num))
			continue;

		/* there is a match - this target is already connected */

		/* There is a rare possibility of a race in the following
		   scenario:
			a. A link goes down,
			b. ib_srp decide to remove the corresponding scsi_host.
			c. Before removing it, the link returns
			d. srp_daemon gets trap 64.
			e. srp_daemon thinks that this target is still
			   connected (ib_srp has not removed it yet) so it
			   does not connect to it.
			f. ib_srp continue to remove the scsi_host.
		    As a result there is no connection to a target in the fabric
		    and there will not be a new trap.

		   To solve this race we schedule here another call to check
		   if this target exist in the near future.
		*/



		/* If there is a need to print all we will continue to pr_cmd.
		   not_connected is set to zero to make sure that this target
		   will be printed but not connected.
		*/
		if (config->all) {
			not_connected = 0;
			break;
		}

		pr_debug("This target is already connected - skip\n");
		closedir(dir);

		return 0;

	}

	len = snprintf(target_config_str, sizeof(target_config_str), "id_ext=%s,"
		"ioc_guid=%016llx,"
		"dgid=%016llx%016llx,"
		"pkey=%04x,"
		"service_id=%016llx",
		target->id_ext,
		(unsigned long long) be64toh(target->ioc_prof.guid),
		(unsigned long long) target->subnet_prefix,
		(unsigned long long) target->h_guid,
		target->pkey,
		(unsigned long long) target->h_service_id);
	if (len >= sizeof(target_config_str)) {
		pr_err("Target config string is too long, ignoring target\n");
		closedir(dir);
		return -1;
	}

	if (target->ioc_prof.io_class != htobe16(SRP_REV16A_IB_IO_CLASS)) {
		len += snprintf(target_config_str+len,
				sizeof(target_config_str) - len,
				",io_class=%04hx", be16toh(target->ioc_prof.io_class));

		if (len >= sizeof(target_config_str)) {
			pr_err("Target config string is too long, ignoring target\n");
			closedir(dir);
			return -1;
		}
	}

	if (config->print_initiator_ext) {
		len += snprintf(target_config_str+len,
				sizeof(target_config_str) - len,
				",initiator_ext=%016llx",
				(unsigned long long) target->h_guid);

		if (len >= sizeof(target_config_str)) {
			pr_err("Target config string is too long, ignoring target\n");
			closedir(dir);
			return -1;
		}
	}

	if (config->execute && config->tl_retry_count) {
		len += snprintf(target_config_str + len,
				sizeof(target_config_str) - len,
				",tl_retry_count=%d", config->tl_retry_count);

		if (len >= sizeof(target_config_str)) {
			pr_err("Target config string is too long, ignoring target\n");
			closedir(dir);
			return -1;
		}
	}

	if (target->options) {
		len += snprintf(target_config_str+len,
				sizeof(target_config_str) - len,
				"%s",
				target->options);

		if (len >= sizeof(target_config_str)) {
			pr_err("Target config string is too long, ignoring target\n");
			closedir(dir);
			return -1;
		}
	}

	/*
	 * The SRP initiator stops parsing parameters if it encounters
	 * an unrecognized parameter. Rest parameters will be ignored.
	 * Append 'max_it_iu_size' in the very end of login string to
	 * avoid breaking SRP login.
	 */
	send_size = be32toh(target->ioc_prof.send_size);
	if (use_imm_data() && imm_data_size_gt_send_size(send_size)) {
		len += snprintf(target_config_str+len,
			sizeof(target_config_str) - len,
			",max_it_iu_size=%d", send_size);

		if (len >= sizeof(target_config_str)) {
			pr_err("Target config string is too long, ignoring target\n");
			closedir(dir);
			return -1;
		}
	}

	target_config_str[len] = '\0';

	pr_cmd(target_config_str, not_connected);

	closedir(dir);

	return 1;
}

static int send_and_get(int portid, int agent, struct srp_ib_user_mad *out_mad,
		 struct srp_ib_user_mad *in_mad, int in_mad_size)
{
	struct umad_dm_packet *out_dm_mad = (void *) out_mad->hdr.data;
	struct umad_dm_packet *in_dm_mad = (void *) in_mad->hdr.data;
	int i, len;
	int in_agent;
	int ret;
	static uint32_t tid;
	uint32_t received_tid;

	for (i = 0; i < config->mad_retries; ++i) {
		/* Skip tid 0 because OpenSM ignores it. */
		if (++tid == 0)
			++tid;
		out_dm_mad->mad_hdr.tid = htobe64(tid);

		ret = umad_send(portid, agent, out_mad, MAD_BLOCK_SIZE,
				config->timeout, 0);
		if (ret < 0) {
			pr_err("umad_send to %u failed\n",
				(uint16_t) be16toh(out_mad->hdr.addr.lid));
			return ret;
		}

		do {
recv:
			len = in_mad_size ? in_mad_size : MAD_BLOCK_SIZE;
			in_agent = umad_recv(portid, (struct ib_user_mad *) in_mad,
					     &len, config->timeout);
			if (in_agent < 0) {
				pr_err("umad_recv from %u failed - %d\n",
					(uint16_t) be16toh(out_mad->hdr.addr.lid),
					in_agent);
				return in_agent;
			}
			if (in_agent != agent) {
				pr_debug("umad_recv returned different agent\n");
				goto recv;
			}

			ret = umad_status(in_mad);
			if (ret) {
				pr_err(
					"bad MAD status (%u) from lid %#x\n",
					ret, be16toh(out_mad->hdr.addr.lid));
				return -ret;
			}

			received_tid = be64toh(in_dm_mad->mad_hdr.tid);
			if (tid != received_tid)
				pr_debug("umad_recv returned different transaction id sent %d got %d\n",
					 tid, received_tid);

		} while ((int32_t)(tid - received_tid) > 0);

		if (len > 0)
			return len;
	}

	return -1;
}

static void initialize_sysfs(void)
{
	char *env;

	env = getenv("SYSFS_PATH");
	if (env) {
		int len;
		char *dup;

		sysfs_path = dup = strndup(env, 256);
		len = strlen(dup);
		while (len > 0 && dup[len - 1] == '/') {
			--len;
			dup[len] = '\0';
		}
	}
}

static int translate_umad_to_ibdev_and_port(char *umad_dev, char **ibdev,
					    char **ibport)
{
	char *class_dev_path;
	char *umad_dev_name;
	int ret;

	*ibdev = NULL;
	*ibport = NULL;

	umad_dev_name = rindex(umad_dev, '/');
	if (!umad_dev_name) {
		pr_err("Couldn't find device name in '%s'\n", umad_dev);
		return -1;
	}

	ret = asprintf(&class_dev_path, "%s/class/infiniband_mad/%s", sysfs_path,
		       umad_dev_name);

	if (ret < 0) {
 		pr_err("out of memory\n");
		return -ENOMEM;
	}

	*ibdev = malloc(IBDEV_STR_SIZE);
	if (!*ibdev) {
 		pr_err("out of memory\n");
		ret = -ENOMEM;
		goto end;
	}

	if (srpd_sys_read_string(class_dev_path, "ibdev", *ibdev,
			    IBDEV_STR_SIZE) < 0) {
		pr_err("Couldn't read ibdev attribute\n");
		ret = -1;
		goto end;
	}

	*ibport = malloc(IBPORT_STR_SIZE);
	if (!*ibport) {
		pr_err("out of memory\n");
		ret = -ENOMEM;
		goto end;
	}
	if (srpd_sys_read_string(class_dev_path, "port", *ibport, IBPORT_STR_SIZE) < 0) {
		pr_err("Couldn't read port attribute\n");
		ret = -1;
		goto end;
	}

	ret = 0;

end:
	if (ret) {
		free(*ibport);
		free(*ibdev);
		*ibdev = NULL;
	}
	free(class_dev_path);

	return ret;
}

static void init_srp_mad(struct srp_ib_user_mad *out_umad, int agent,
			 uint16_t h_dlid, uint16_t h_attr_id, uint32_t h_attr_mod)
{
	struct umad_dm_packet *out_mad;

	memset(out_umad, 0, sizeof *out_umad);

	out_umad->hdr.agent_id   = agent;
	out_umad->hdr.addr.qpn   = htobe32(1);
	out_umad->hdr.addr.qkey  = htobe32(UMAD_QKEY);
	out_umad->hdr.addr.lid   = htobe16(h_dlid);

	out_mad = (void *) out_umad->hdr.data;

	out_mad->mad_hdr.base_version  = UMAD_BASE_VERSION;
	out_mad->mad_hdr.method        = UMAD_METHOD_GET;
	out_mad->mad_hdr.attr_id       = htobe16(h_attr_id);
	out_mad->mad_hdr.attr_mod      = htobe32(h_attr_mod);
}

static void init_srp_dm_mad(struct srp_ib_user_mad *out_mad, int agent, uint16_t h_dlid,
			    uint16_t h_attr_id, uint32_t h_attr_mod)
{
	struct umad_sa_packet *out_dm_mad = get_data_ptr(*out_mad);

	init_srp_mad(out_mad, agent, h_dlid, h_attr_id, h_attr_mod);
	out_dm_mad->mad_hdr.mgmt_class = UMAD_CLASS_DEVICE_MGMT;
	out_dm_mad->mad_hdr.class_version  = 1;
}

static void init_srp_sa_mad(struct srp_ib_user_mad *out_mad, int agent, uint16_t h_dlid,
			    uint16_t h_attr_id, uint32_t h_attr_mod)
{
	struct umad_sa_packet *out_sa_mad = get_data_ptr(*out_mad);

	init_srp_mad(out_mad, agent, h_dlid, h_attr_id, h_attr_mod);
	out_sa_mad->mad_hdr.mgmt_class = UMAD_CLASS_SUBN_ADM;
	out_sa_mad->mad_hdr.class_version  = UMAD_SA_CLASS_VERSION;
}

static int check_sm_cap(struct umad_resources *umad_res, int *mask_match)
{
	struct srp_ib_user_mad		out_mad, in_mad;
	struct umad_sa_packet	       *in_sa_mad;
	struct umad_class_port_info    *cpi;
	int				ret;

	in_sa_mad  = get_data_ptr(in_mad);

	init_srp_sa_mad(&out_mad, umad_res->agent, umad_res->sm_lid,
		        UMAD_ATTR_CLASS_PORT_INFO, 0);

	ret = send_and_get(umad_res->portid, umad_res->agent, &out_mad, &in_mad, 0);
	if (ret < 0)
		return ret;

	cpi = (void *) in_sa_mad->data;

	*mask_match = !!(be16toh(cpi->cap_mask) & SRP_SM_SUPPORTS_MASK_MATCH);

	return 0;
}

int pkey_index_to_pkey(struct umad_resources *umad_res, int pkey_index,
		       __be16 *pkey)
{
	if (ibv_query_pkey(umad_res->ib_ctx, config->port_num, pkey_index,
			   pkey) < 0)
		return -1;
	if (*pkey)
		pr_debug("discover Targets for P_key %04x (index %d)\n",
			 *pkey, pkey_index);
	return 0;
}

static int pkey_to_pkey_index(struct umad_resources *umad_res, uint16_t h_pkey,
			      uint16_t *pkey_index)
{
	int res = ibv_get_pkey_index(umad_res->ib_ctx, config->port_num,
				     htobe16(h_pkey));
	if (res >= 0)
		*pkey_index = res;
	return res;
}

static int set_class_port_info(struct umad_resources *umad_res, uint16_t dlid, uint16_t h_pkey)
{
	struct srp_ib_user_mad		in_mad, out_mad;
	struct umad_dm_packet	       *out_dm_mad, *in_dm_mad;
	struct umad_class_port_info    *cpi;
	char val[64];
	int i;

	init_srp_dm_mad(&out_mad, umad_res->agent, dlid, UMAD_ATTR_CLASS_PORT_INFO, 0);

	if (pkey_to_pkey_index(umad_res, h_pkey, &out_mad.hdr.addr.pkey_index)
	    < 0) {
		pr_err("set_class_port_info: Unable to find pkey_index for pkey %#x\n", h_pkey);
		return -1;
	}

	out_dm_mad = get_data_ptr(out_mad);
	out_dm_mad->mad_hdr.method = UMAD_METHOD_SET;

	cpi                = (void *) out_dm_mad->data;

	if (srpd_sys_read_string(umad_res->port_sysfs_path, "lid", val, sizeof val) < 0) {
		pr_err("Couldn't read LID\n");
		return -1;
	}

	cpi->trap_lid = htobe16(strtol(val, NULL, 0));

	if (srpd_sys_read_string(umad_res->port_sysfs_path, "gids/0", val, sizeof val) < 0) {
		pr_err("Couldn't read GID[0]\n");
		return -1;
	}

	for (i = 0; i < 8; ++i)
		cpi->trapgid.raw_be16[i] = htobe16(strtol(val + i * 5, NULL, 16));

	if (send_and_get(umad_res->portid, umad_res->agent, &out_mad, &in_mad, 0) < 0)
		return -1;

	in_dm_mad = get_data_ptr(in_mad);
	if (in_dm_mad->mad_hdr.status) {
		pr_err("Class Port Info set returned status 0x%04x\n",
			be16toh(in_dm_mad->mad_hdr.status));
		return -1;
	}

	return 0;
}

static int get_iou_info(struct umad_resources *umad_res, uint16_t dlid,
			uint16_t h_pkey, struct srp_dm_iou_info *iou_info)
{
	struct srp_ib_user_mad		in_mad, out_mad;
	struct umad_dm_packet	       *in_dm_mad;

	init_srp_dm_mad(&out_mad, umad_res->agent, dlid, SRP_DM_ATTR_IO_UNIT_INFO, 0);
	if (pkey_to_pkey_index(umad_res, h_pkey, &out_mad.hdr.addr.pkey_index)
	    < 0) {
		pr_err("get_iou_info: Unable to find pkey_index for pkey %#x\n", h_pkey);
		return -1;
	}

	if (send_and_get(umad_res->portid, umad_res->agent, &out_mad, &in_mad, 0) < 0)
		return -1;

	in_dm_mad = get_data_ptr(in_mad);
	if (in_dm_mad->mad_hdr.status) {
		pr_err("IO Unit Info query returned status 0x%04x\n",
			be16toh(in_dm_mad->mad_hdr.status));
		return -1;
	}

	memcpy(iou_info, in_dm_mad->data, sizeof *iou_info);
/*
	pr_debug("iou_info->max_controllers is %d\n", iou_info->max_controllers);
*/
	return 0;
}

static int get_ioc_prof(struct umad_resources *umad_res, uint16_t h_dlid, uint16_t h_pkey, int ioc,
			struct srp_dm_ioc_prof *ioc_prof)
{
	struct srp_ib_user_mad		in_mad, out_mad;
	struct umad_dm_packet	       *in_dm_mad;

	init_srp_dm_mad(&out_mad, umad_res->agent, h_dlid, SRP_DM_ATTR_IO_CONTROLLER_PROFILE, ioc);

	if (pkey_to_pkey_index(umad_res, h_pkey, &out_mad.hdr.addr.pkey_index)
	    < 0) {
		pr_err("get_ioc_prof: Unable to find pkey_index for pkey %#x\n",
		       h_pkey);
		return -1;
	}

	if (send_and_get(umad_res->portid, umad_res->agent, &out_mad, &in_mad, 0) < 0)
		return -1;

	in_dm_mad = get_data_ptr(in_mad);
	if (in_dm_mad->mad_hdr.status) {
		pr_err("IO Controller Profile query returned status 0x%04x for %d\n",
			be16toh(in_dm_mad->mad_hdr.status), ioc);
		return -1;
	}

	memcpy(ioc_prof, in_dm_mad->data, sizeof *ioc_prof);

	return 0;
}

static int get_svc_entries(struct umad_resources *umad_res, uint16_t dlid, uint16_t h_pkey, int ioc,
			   int start, int end, struct srp_dm_svc_entries *svc_entries)
{
	struct srp_ib_user_mad		in_mad, out_mad;
	struct umad_dm_packet	       *in_dm_mad;

	init_srp_dm_mad(&out_mad, umad_res->agent, dlid, SRP_DM_ATTR_SERVICE_ENTRIES,
			(ioc << 16) | (end << 8) | start);

	if (pkey_to_pkey_index(umad_res, h_pkey, &out_mad.hdr.addr.pkey_index)
	    < 0) {
		pr_err("get_svc_entries: Unable to find pkey_index for pkey %#x\n",
		       h_pkey);
		return -1;
	}

	if (send_and_get(umad_res->portid, umad_res->agent, &out_mad, &in_mad, 0) < 0)
		return -1;

	in_dm_mad = get_data_ptr(in_mad);
	if (in_dm_mad->mad_hdr.status) {
		pr_err("Service Entries query returned status 0x%04x\n",
			be16toh(in_dm_mad->mad_hdr.status));
		return -1;
	}

	memcpy(svc_entries, in_dm_mad->data, sizeof *svc_entries);

	return 0;
}

static int do_port(struct resources *res, uint16_t pkey, uint16_t dlid,
		   uint64_t subnet_prefix, uint64_t h_guid)
{
	struct umad_resources 	       *umad_res = res->umad_res;
	struct srp_dm_iou_info		iou_info;
	struct srp_dm_svc_entries	svc_entries;
	int				i, j, k, ret;

	static const uint64_t topspin_oui = 0x0005ad0000000000ull;
	static const uint64_t oui_mask    = 0xffffff0000000000ull;

	struct target_details *target = (struct target_details *)
		malloc(sizeof(struct target_details));

	target->subnet_prefix = subnet_prefix;
	target->h_guid = h_guid;
	target->options = NULL;

 	pr_debug("enter do_port\n");
	if ((target->h_guid & oui_mask) == topspin_oui &&
	    set_class_port_info(umad_res, dlid, pkey))
		pr_err("Warning: set of ClassPortInfo failed\n");

	ret = get_iou_info(umad_res, dlid, pkey, &iou_info);
	if (ret < 0) {
		pr_err("failed to get iou info for dlid %#x\n", dlid);
		goto out;
	}

	pr_human("IO Unit Info:\n");
	pr_human("    port LID:        %04x\n", dlid);
	pr_human("    port GID:        %016llx%016llx\n",
		 (unsigned long long) target->subnet_prefix,
		 (unsigned long long) target->h_guid);
	pr_human("    change ID:       %04x\n", be16toh(iou_info.change_id));
	pr_human("    max controllers: 0x%02x\n", iou_info.max_controllers);

	if (config->verbose > 0)
		for (i = 0; i < iou_info.max_controllers; ++i) {
			pr_human("    controller[%3d]: ", i + 1);
			switch ((iou_info.controller_list[i / 2] >>
				 (4 * (1 - i % 2))) & 0xf) {
			case SRP_DM_NO_IOC:      pr_human("not installed\n"); break;
			case SRP_DM_IOC_PRESENT: pr_human("present\n");       break;
			case SRP_DM_NO_SLOT:     pr_human("no slot\n");       break;
			default:                 pr_human("<unknown>\n");     break;
			}
		}

	for (i = 0; i < iou_info.max_controllers; ++i) {
		if (((iou_info.controller_list[i / 2] >> (4 * (1 - i % 2))) & 0xf) ==
		    SRP_DM_IOC_PRESENT) {
			pr_human("\n");

			if (get_ioc_prof(umad_res, dlid, pkey, i + 1, &target->ioc_prof))
				continue;

			pr_human("    controller[%3d]\n", i + 1);

			pr_human("        GUID:      %016llx\n",
				 (unsigned long long) be64toh(target->ioc_prof.guid));
			pr_human("        vendor ID: %06x\n", be32toh(target->ioc_prof.vendor_id) >> 8);
			pr_human("        device ID: %06x\n", be32toh(target->ioc_prof.device_id));
			pr_human("        IO class : %04hx\n", be16toh(target->ioc_prof.io_class));
			pr_human("        Maximum size of Send Messages in bytes: %d\n",
				 be32toh(target->ioc_prof.send_size));
			pr_human("        ID:        %s\n", target->ioc_prof.id);
			pr_human("        service entries: %d\n", target->ioc_prof.service_entries);

			for (j = 0; j < target->ioc_prof.service_entries; j += 4) {
				int n;

				n = j + 3;
				if (n >= target->ioc_prof.service_entries)
					n = target->ioc_prof.service_entries - 1;

				if (get_svc_entries(umad_res, dlid, pkey, i + 1,
						    j, n, &svc_entries))
					continue;

				for (k = 0; k <= n - j; ++k) {

					if (sscanf(svc_entries.service[k].name,
						   "SRP.T10:%16s",
						   target->id_ext) != 1)
						continue;

					pr_human("            service[%3d]: %016llx / %s\n",
						 j + k,
						 (unsigned long long) be64toh(svc_entries.service[k].id),
						 svc_entries.service[k].name);

					target->h_service_id = be64toh(svc_entries.service[k].id);
					target->pkey = pkey;
					if (is_enabled_by_rules_file(target)) {
						if (!add_non_exist_target(target) && !config->once) {
							target->retry_time =
								time(NULL) + config->retry_timeout;
							push_to_retry_list(res->sync_res, target);
						}
					}
				}
			}
		}
	}

	pr_human("\n");

out:
	free(target);
	return ret;
}

int get_node(struct umad_resources *umad_res, uint16_t dlid, uint64_t *guid)
{
	struct srp_ib_user_mad		out_mad, in_mad;
	struct umad_sa_packet	       *out_sa_mad, *in_sa_mad;
	struct srp_sa_node_rec	       *node;

	in_sa_mad = get_data_ptr(in_mad);
	out_sa_mad = get_data_ptr(out_mad);

	init_srp_sa_mad(&out_mad, umad_res->agent, umad_res->sm_lid,
		        UMAD_SA_ATTR_NODE_REC, 0);

	out_sa_mad->comp_mask     = htobe64(1); /* LID */
	node			  = (void *) out_sa_mad->data;
	node->lid		  = htobe16(dlid);

	if (send_and_get(umad_res->portid, umad_res->agent, &out_mad, &in_mad, 0) < 0)
		return -1;

	node  = (void *) in_sa_mad->data;
	*guid = be64toh(node->port_guid);

	return 0;
}

static int get_port_info(struct umad_resources *umad_res, uint16_t dlid,
			 uint64_t *subnet_prefix, int *isdm)
{
	struct srp_ib_user_mad		out_mad, in_mad;
	struct umad_sa_packet	       *out_sa_mad, *in_sa_mad;
	struct srp_sa_port_info_rec    *port_info;

	in_sa_mad = get_data_ptr(in_mad);
	out_sa_mad = get_data_ptr(out_mad);

	init_srp_sa_mad(&out_mad, umad_res->agent, umad_res->sm_lid,
		        UMAD_SA_ATTR_PORT_INFO_REC, 0);

	out_sa_mad->comp_mask     = htobe64(1); /* LID */
	port_info                 = (void *) out_sa_mad->data;
	port_info->endport_lid	  = htobe16(dlid);

	if (send_and_get(umad_res->portid, umad_res->agent, &out_mad, &in_mad, 0) < 0)
		return -1;

	port_info = (void *) in_sa_mad->data;
	*subnet_prefix = be64toh(port_info->subnet_prefix);
	*isdm          = !!(be32toh(port_info->capability_mask) & SRP_IS_DM);

	return 0;
}

static int get_shared_pkeys(struct resources *res,
			    uint16_t dest_port_lid,
			    uint16_t *pkeys)
{
	struct umad_resources          *umad_res = res->umad_res;
	uint8_t                        *in_mad_buf;
	struct srp_ib_user_mad		out_mad;
	struct ib_user_mad	       *in_mad;
	struct umad_sa_packet	       *out_sa_mad, *in_sa_mad;
	struct ib_path_rec	       *path_rec;
	ssize_t len;
	int i, num_pkeys = 0;
	__be16 pkey;
	uint16_t local_port_lid = get_port_lid(res->ud_res->ib_ctx,
					       config->port_num, NULL);

	in_mad_buf = malloc(sizeof(struct ib_user_mad) +
			    node_table_response_size);
	if (!in_mad_buf)
		return -ENOMEM;

	in_mad = (void *)in_mad_buf;
	in_sa_mad = (void *)in_mad->data;
	out_sa_mad = get_data_ptr(out_mad);

	init_srp_sa_mad(&out_mad, umad_res->agent, umad_res->sm_lid,
		        UMAD_SA_ATTR_PATH_REC, 0);

	/**
	 * Due to OpenSM bug (issue #335016) SM won't return
	 * table of all shared P_Keys, it will return only the first
	 * shared P_Key, So we send path_rec over each P_Key in the P_Key
	 * table. SM will return path record if P_Key is shared or else None.
	 * Once SM bug will be fixed, this loop should be removed.
	 **/
	for (i = 0; ; i++) {
		if (pkey_index_to_pkey(umad_res, i, &pkey))
			break;
		if (!pkey)
			continue;

		/* Mark components: DLID, SLID, PKEY */
		out_sa_mad->comp_mask = htobe64(1 << 4 | 1 << 5 | 1 << 13);
		path_rec = (struct ib_path_rec *)out_sa_mad->data;
		path_rec->slid = htobe16(local_port_lid);
		path_rec->dlid = htobe16(dest_port_lid);
		path_rec->pkey = pkey;

		len = send_and_get(umad_res->portid, umad_res->agent, &out_mad,
				   (struct srp_ib_user_mad *)in_mad,
				   node_table_response_size);
		if (len < 0)
			goto err;

		path_rec = (struct ib_path_rec *)in_sa_mad->data;
		pkeys[num_pkeys++] = be16toh(path_rec->pkey);
	}

	free(in_mad_buf);
	return num_pkeys;
err:
	free(in_mad_buf);
	return -1;
}

static int do_dm_port_list(struct resources *res)
{
	struct umad_resources 	       *umad_res = res->umad_res;
	uint8_t                        *in_mad_buf;
	struct srp_ib_user_mad		out_mad;
	struct ib_user_mad	       *in_mad;
	struct umad_sa_packet	       *out_sa_mad, *in_sa_mad;
	struct srp_sa_port_info_rec    *port_info;
	ssize_t len;
	int size;
	int i, j,num_pkeys;
	uint16_t pkeys[SRP_MAX_SHARED_PKEYS];
	uint64_t guid;

	in_mad_buf = malloc(sizeof(struct ib_user_mad) +
			    node_table_response_size);
	if (!in_mad_buf)
		return -ENOMEM;

	in_mad     = (void *) in_mad_buf;
	in_sa_mad  = (void *) in_mad->data;
	out_sa_mad = get_data_ptr(out_mad);

	init_srp_sa_mad(&out_mad, umad_res->agent, umad_res->sm_lid,
		        UMAD_SA_ATTR_PORT_INFO_REC, SRP_SM_CAP_MASK_MATCH_ATTR_MOD);

	out_sa_mad->mad_hdr.method = UMAD_SA_METHOD_GET_TABLE;
	out_sa_mad->comp_mask      = htobe64(1 << 7); /* Capability mask */
	out_sa_mad->rmpp_hdr.rmpp_version = UMAD_RMPP_VERSION;
	out_sa_mad->rmpp_hdr.rmpp_type = 1;
	port_info		   = (void *) out_sa_mad->data;
	port_info->capability_mask = htobe32(SRP_IS_DM); /* IsDM */

	len = send_and_get(umad_res->portid, umad_res->agent, &out_mad,
			   (struct srp_ib_user_mad *) in_mad,
			   node_table_response_size);
	if (len < 0) {
		free(in_mad_buf);
		return len;
	}

	size = ib_get_attr_size(in_sa_mad->attr_offset);
	if (!size) {
		if (config->verbose) {
			printf("Query did not find any targets\n");
		}
		free(in_mad_buf);
		return 0;
	}

	for (i = 0; (i + 1) * size <= len - MAD_RMPP_HDR_SIZE; ++i) {
		port_info = (void *) in_sa_mad->data + i * size;
		if (get_node(umad_res, be16toh(port_info->endport_lid), &guid))
			continue;

		num_pkeys = get_shared_pkeys(res, be16toh(port_info->endport_lid),
					     pkeys);
		if (num_pkeys < 0) {
			pr_err("failed to get shared P_Keys with LID %#x\n",
			       be16toh(port_info->endport_lid));
			free(in_mad_buf);
			return num_pkeys;
		}

		for (j = 0; j < num_pkeys; ++j)
			do_port(res, pkeys[j], be16toh(port_info->endport_lid),
				be64toh(port_info->subnet_prefix), guid);
	}

	free(in_mad_buf);
	return 0;
}

void handle_port(struct resources *res, uint16_t pkey, uint16_t lid, uint64_t h_guid)
{
	struct umad_resources *umad_res = res->umad_res;
	uint64_t subnet_prefix;
	int isdm;

	pr_debug("enter handle_port for lid %#x\n", lid);
	if (get_port_info(umad_res, lid, &subnet_prefix, &isdm))
		return;

	if (!isdm)
		return;

	do_port(res, pkey, lid, subnet_prefix, h_guid);
}


static int do_full_port_list(struct resources *res)
{
	struct umad_resources 	       *umad_res = res->umad_res;
	uint8_t                        *in_mad_buf;
	struct srp_ib_user_mad		out_mad;
	struct ib_user_mad	       *in_mad;
	struct umad_sa_packet	       *out_sa_mad, *in_sa_mad;
	struct srp_sa_node_rec	       *node;
	ssize_t len;
	int size;
	int i, j, num_pkeys;
	uint16_t pkeys[SRP_MAX_SHARED_PKEYS];

	in_mad_buf = malloc(sizeof(struct ib_user_mad) +
			    node_table_response_size);
	if (!in_mad_buf)
		return -ENOMEM;

	in_mad     = (void *) in_mad_buf;
	in_sa_mad  = (void *) in_mad->data;
	out_sa_mad = get_data_ptr(out_mad);

	init_srp_sa_mad(&out_mad, umad_res->agent, umad_res->sm_lid,
		        UMAD_SA_ATTR_NODE_REC, 0);

	out_sa_mad->mad_hdr.method = UMAD_SA_METHOD_GET_TABLE;
	out_sa_mad->comp_mask     = 0; /* Get all end ports */
	out_sa_mad->rmpp_hdr.rmpp_version = UMAD_RMPP_VERSION;
	out_sa_mad->rmpp_hdr.rmpp_type = 1;

	len = send_and_get(umad_res->portid, umad_res->agent, &out_mad,
			   (struct srp_ib_user_mad *) in_mad,
			   node_table_response_size);
	if (len < 0) {
		free(in_mad_buf);
		return len;
	}

	size = be16toh(in_sa_mad->attr_offset) * 8;

	for (i = 0; (i + 1) * size <= len - MAD_RMPP_HDR_SIZE; ++i) {
		node = (void *) in_sa_mad->data + i * size;

		num_pkeys = get_shared_pkeys(res, be16toh(node->lid),
					     pkeys);
		if (num_pkeys < 0) {
			pr_err("failed to get shared P_Keys with LID %#x\n",
			       be16toh(node->lid));
			free(in_mad_buf);
			return num_pkeys;
		}

		for (j = 0; j < num_pkeys; ++j)
			(void) handle_port(res, pkeys[j], be16toh(node->lid),
					   be64toh(node->port_guid));
	}

	free(in_mad_buf);
	return 0;
}

struct config_t *config;

static void print_config(struct config_t *conf)
{
	printf(" configuration report\n");
	printf(" ------------------------------------------------\n");
	printf(" Current pid                		: %u\n", getpid());
	printf(" Device name                		: \"%s\"\n", conf->dev_name);
	printf(" IB port                    		: %u\n", conf->port_num);
	printf(" Mad Retries                		: %d\n", conf->mad_retries);
	printf(" Number of outstanding WR   		: %u\n", conf->num_of_oust);
	printf(" Mad timeout (msec)	     		: %u\n", conf->timeout);
	printf(" Prints add target command  		: %d\n", conf->cmd);
 	printf(" Executes add target command		: %d\n", conf->execute);
 	printf(" Print also connected targets 		: %d\n", conf->all);
 	printf(" Report current targets and stop 	: %d\n", conf->once);
	if (conf->rules_file)
		printf(" Reads rules from 			: %s\n", conf->rules_file);
	if (conf->print_initiator_ext)
		printf(" Print initiator_ext\n");
	else
		printf(" Do not print initiator_ext\n");
	if (conf->recalc_time)
		printf(" Performs full target rescan every %d seconds\n", conf->recalc_time);
	else
		printf(" No full target rescan\n");
	if (conf->retry_timeout)
		printf(" Retries to connect to existing target after %d seconds\n", conf->retry_timeout);
	else
		printf(" Do not retry to connect to existing targets\n");
	printf(" ------------------------------------------------\n");
}

static char *copy_till_comma(char *d, char *s, int len, int base)
{
	int i=0;

	while (strchr(", \t\n", *s) == NULL) {
		if (i == len)
			return NULL;
		if ((base == 16 && isxdigit(*s)) || (base == 10 && isdigit(*s))) {
			*d=*s;
			++d;
			++s;
			++i;
		} else
			return NULL;
	}
	*d='\0';

	if (*s == '\n')
		return s;

	++s;
	return s;
}

static char *parse_main_option(struct rule *rule, char *ptr)
{
	struct option_info {
		const char *name;
		size_t offset;
		size_t len;
		int base;
	};
#define OPTION_INFO(n, base) { #n "=", offsetof(struct rule, n),	\
			sizeof(((struct rule *)NULL)->n), base}
	static const struct option_info opt_info[] = {
		OPTION_INFO(id_ext, 16),
		OPTION_INFO(ioc_guid, 16),
		OPTION_INFO(dgid, 16),
		OPTION_INFO(service_id, 16),
		OPTION_INFO(pkey, 16),
	};
	int i, optnamelen;
	char *ptr2 = NULL;

	for (i = 0; i < sizeof(opt_info) / sizeof(opt_info[0]); i++) {
		optnamelen = strlen(opt_info[i].name);
		if (strncmp(ptr, opt_info[i].name, optnamelen) == 0) {
			ptr2 = copy_till_comma((char *)rule
					       + opt_info[i].offset,
					       ptr + optnamelen,
					       opt_info[i].len - 1,
					       opt_info[i].base);
			break;
		}
	}

	return ptr2;
}

/*
 * Return values:
 *  -1 if the output buffer is not large enough.
 *   0 if an unsupported option has been encountered.
 * > 0 if parsing succeeded.
 */
static int parse_other_option(struct rule *rule, char *ptr)
{
	static const char *const opt[] = {
		"allow_ext_sg=",
		"ch_count=",
		"cmd_sg_entries=",
		"comp_vector=",
		"max_cmd_per_lun=",
		"max_sect=",
		"queue_size=",
		"sg_tablesize=",
		"tl_retry_count=",
	};

	char *ptr2 = NULL, *optr, option[17];
	int i, optnamelen, len, left;

	optr = rule->options;
	left = sizeof(rule->options);
	len = strlen(optr);
	optr += len;
	left -= len;
	for (i = 0; i < sizeof(opt)/sizeof(opt[0]); ++i) {
		optnamelen = strlen(opt[i]);
		if (strncmp(ptr, opt[i], optnamelen) != 0)
			continue;
		ptr2 = copy_till_comma(option, ptr + optnamelen,
				       sizeof(option) - 1, 10);
		if (!ptr2)
			return -1;
		len = snprintf(optr, left, ",%s%s", opt[i], option);
		optr += len;
		left -= len;
		if (left <= 0)
			return -1;
		break;
	}
	return ptr2 ? ptr2 - ptr : 0;
}

static int get_rules_file(struct config_t *conf)
{
	int line_number = 1, len, line_number_for_output, ret = -1;
	char line[255];
	char *ptr, *ptr2;
	struct rule *rule;
	FILE *infile = fopen(conf->rules_file, "r");

	if (infile == NULL) {
		pr_debug("Could not find rules file %s, going with default\n",
			 conf->rules_file);
		return 0;
	}

	while (fgets(line, sizeof(line), infile) != NULL) {
		if (line[0] != '#' && line[0] != '\n')
			line_number++;
	}

	if (fseek(infile, 0L, SEEK_SET) != 0) {
		pr_err("internal error while seeking %s\n", conf->rules_file);
		goto out;
	}

	conf->rules = malloc(sizeof(struct rule) * line_number);

	rule = &conf->rules[0] - 1;
	line_number_for_output = 0;
	while (fgets(line, sizeof(line), infile) != NULL) {
		line_number_for_output++;
		if (line[0] == '#' || line[0] == '\n')
			continue;

		rule++;
		switch (line[0]) {
		case 'a':
		case 'A':
			rule->allow = 1;
			break;
		case 'd':
		case 'D':
			rule->allow = 0;
			break;
		default:
			pr_err("Bad syntax in rules file %s line %d:"
			       " line should start with 'a' or 'd'\n",
			       conf->rules_file, line_number_for_output);
			goto out;
		}

		rule->id_ext[0] = '\0';
		rule->ioc_guid[0] = '\0';
		rule->dgid[0] = '\0';
		rule->service_id[0] = '\0';
		rule->pkey[0] = '\0';
		rule->options[0] = '\0';

		ptr = &line[1];
		while (*ptr == ' ' || *ptr == '\t')
			ptr++;

		while (*ptr != '\n') {
			ptr2 = parse_main_option(rule, ptr);
			if (!ptr2 && rule->allow) {
				len = parse_other_option(rule, ptr);
				if (len < 0) {
					pr_err("Buffer overflow triggered by"
					       " rules file %s line %d\n",
					       conf->rules_file,
					       line_number_for_output);
					goto out;
				}
				ptr2 = len ? ptr + len : NULL;
			}

			if (ptr2 == NULL) {
				pr_err("Bad syntax in rules file %s line %d\n",
				       conf->rules_file, line_number_for_output);
				goto out;
			}
			ptr = ptr2;

			while (*ptr == ' ' || *ptr == '\t')
				ptr++;
		}
	}
	rule++;
	rule->id_ext[0] = '\0';
	rule->ioc_guid[0] = '\0';
	rule->dgid[0] = '\0';
	rule->service_id[0] = '\0';
	rule->pkey[0] = '\0';
	rule->options[0] = '\0';
	rule->allow = 1;
	ret = 0;

out:
	fclose(infile);

	return ret;
}

static int set_conf_dev_and_port(char *umad_dev, struct config_t *conf)
{
	int ret;

	if (umad_dev) {
		char *ibport;

		ret = translate_umad_to_ibdev_and_port(umad_dev,
						       &conf->dev_name,
						       &ibport);
		if (ret) {
			pr_err("Fail to translate umad to ibdev and port\n");
			goto out;
		}
		conf->port_num = atoi(ibport);
		if (conf->port_num == 0) {
			pr_err("Bad port number %s\n", ibport);
			ret = -1;
		}
		free(ibport);
	} else {
		umad_ca_t ca;
		umad_port_t port;

		ret = umad_get_ca(NULL, &ca);
		if (ret) {
			pr_err("Failed to get default CA\n");
			goto out;
		}

		ret = umad_get_port(ca.ca_name, 0, &port);
		if (ret) {
			pr_err("Failed to get default port for CA %s\n",
			       ca.ca_name);
			umad_release_ca(&ca);
			goto out;
		}
		conf->dev_name = strdup(ca.ca_name);
		conf->port_num = port.portnum;
		umad_release_port(&port);
		umad_release_ca(&ca);
		pr_debug("Using device %s port %d\n", conf->dev_name,
			 conf->port_num);
	}
out:
	return ret;
}

static const struct option long_opts[] = {
	{ "systemd",        0, NULL, 'S' },
	{}
};
static const char short_opts[] = "caveod:i:j:p:t:r:R:T:l:Vhnf:";

/* Check if the --systemd options was passed in very early so we can setup
 * logging properly.
 */
static bool is_systemd(int argc, char *argv[])
{
	while (1) {
		int c;

		c = getopt_long(argc, argv, short_opts, long_opts, NULL);
		if (c == -1)
			break;
		if (c == 'S')
			return true;

	}
	return false;
}

static int get_config(struct config_t *conf, int argc, char *argv[])
{
	/* set defaults */
	char* umad_dev = NULL;
	int ret;

	conf->port_num			= 1;
	conf->num_of_oust		= 10;
	conf->dev_name	 		= NULL;
	conf->cmd	 		= 0;
	conf->once	 		= 0;
	conf->execute	 		= 0;
	conf->all	 		= 0;
	conf->verbose	 		= 0;
	conf->debug_verbose    		= 0;
	conf->timeout	 		= 5000;
	conf->mad_retries 		= 3;
	conf->recalc_time 		= 0;
	conf->retry_timeout 		= 20;
	conf->add_target_file  		= NULL;
	conf->print_initiator_ext	= 0;
	conf->rules_file		= SRP_DAEMON_CONFIG_FILE;
	conf->rules			= NULL;
	conf->tl_retry_count		= 0;

	optind = 1;
	while (1) {
		int c;

		c = getopt_long(argc, argv, short_opts, long_opts, NULL);
		if (c == -1)
			break;

		switch (c) {
		case 'd':
			umad_dev = optarg;
			break;
		case 'i':
			conf->dev_name = strdup(optarg);
			if (!conf->dev_name) {
				pr_err("Fail to alloc space for dev_name\n");
				return -ENOMEM;
			}
			break;
		case 'p':
			conf->port_num = atoi(optarg);
			if (conf->port_num == 0) {
				pr_err("Bad port number %s\n", optarg);
				return -1;
			}
			break;
		case 'j': {
			char dev[32];
			int port_num;

			if (sscanf(optarg, "%31[^:]:%d", dev, &port_num) != 2) {
				pr_err("Bad dev:port specification %s\n",
				       optarg);
				return -1;
			}
			conf->dev_name = strdup(dev);
			conf->port_num = port_num;
			}
			break;
		case 'c':
			++conf->cmd;
			break;
		case 'o':
			++conf->once;
			break;
		case 'a':
			++conf->all;
			break;
		case 'e':
			++conf->execute;
			break;
		case 'v':
			++conf->verbose;
			break;
		case 'V':
			++conf->debug_verbose;
			break;
		case 'n':
			++conf->print_initiator_ext;
			break;
		case 't':
			conf->timeout = atoi(optarg);
			if (conf->timeout == 0) {
				pr_err("Bad timeout - %s\n", optarg);
				return -1;
			}
			break;
		case 'r':
			conf->mad_retries = atoi(optarg);
			if (conf->mad_retries == 0) {
				pr_err("Bad number of retries - %s\n", optarg);
				return -1;
			}
			break;
		case 'R':
			conf->recalc_time = atoi(optarg);
			if (conf->recalc_time == 0) {
				pr_err("Bad Rescan time window - %s\n", optarg);
				return -1;
			}
			break;
		case 'T':
			conf->retry_timeout = atoi(optarg);
			if (conf->retry_timeout == 0 && strcmp(optarg, "0")) {
				pr_err("Bad retry Timeout value- %s.\n", optarg);
				return -1;
			}
			break;
		case 'f':
			conf->rules_file = optarg;
			break;
		case 'l':
			conf->tl_retry_count = atoi(optarg);
			if (conf->tl_retry_count < 2 ||
			    conf->tl_retry_count > 7) {
				pr_err("Bad tl_retry_count argument (%d), "
				       "must be 2 <= tl_retry_count <= 7\n",
				       conf->tl_retry_count);
				return -1;
			}
			break;
		case 'S':
			break;
		case 'h':
		default:
			usage(argv[0]);
			return -1;
		}
	}

	initialize_sysfs();

	if (conf->dev_name == NULL) {
		ret = set_conf_dev_and_port(umad_dev, conf);
	        if (ret) {
	                pr_err("Failed to build config\n");
	                return ret;
	        }
	}
	ret = asprintf(&conf->add_target_file,
		       "%s/class/infiniband_srp/srp-%s-%d/add_target", sysfs_path,
		       conf->dev_name, conf->port_num);
	if (ret < 0) {
		pr_err("error while allocating add_target\n");
		return ret;
	}

	if (get_rules_file(conf))
		return -1;

	return 0;
}

static void free_config(struct config_t *conf)
{
	free(conf->dev_name);
	free(conf->add_target_file);
	free(conf->rules);
	free(conf);
}

static void umad_resources_init(struct umad_resources *umad_res)
{
	umad_res->portid = -1;
	umad_res->agent = -1;
	umad_res->port_sysfs_path = NULL;
}

static void umad_resources_destroy(struct umad_resources *umad_res)
{
	if (umad_res->port_sysfs_path)
		free(umad_res->port_sysfs_path);

	if (umad_res->portid >= 0) {
		if (umad_res->agent >= 0)
			umad_unregister(umad_res->portid, umad_res->agent);
		umad_close_port(umad_res->portid);
	}

	umad_done();
}

static int check_link_layer(const char *port_sysfs_path)
{
	const char expected_link_layer[] = "InfiniBand";
	char link_layer[sizeof(expected_link_layer)];
	int ret;

	ret = srpd_sys_read_string(port_sysfs_path, "link_layer", link_layer,
				   sizeof(link_layer));
	if (ret < 0) {
		pr_err("Couldn't read link layer\n");
		return ret;
	}

	if (strcmp(link_layer, expected_link_layer)) {
		pr_err("Unsupported link layer %s\n", link_layer);
		return -EINVAL;
	}

	return 0;
}

static int umad_resources_create(struct umad_resources *umad_res)
{

	int ret;

	ret = asprintf(&umad_res->port_sysfs_path, "%s/class/infiniband/%s/ports/%d",
		       sysfs_path, config->dev_name, config->port_num);

	if (ret < 0) {
		umad_res->port_sysfs_path = NULL;
		return -ENOMEM;
	}

	ret = check_link_layer(umad_res->port_sysfs_path);
	if (ret)
		return ret;

	umad_res->portid = umad_open_port(config->dev_name, config->port_num);
	if (umad_res->portid < 0) {
		pr_err("umad_open_port failed for device %s port %d\n",
		       config->dev_name, config->port_num);
		return -ENXIO;
	}

	umad_res->agent = umad_register(umad_res->portid, UMAD_CLASS_SUBN_ADM,
					   UMAD_SA_CLASS_VERSION,
					   UMAD_RMPP_VERSION, NULL);
	if (umad_res->agent < 0) {
		pr_err("umad_register failed\n");
		return umad_res->agent;
	}

	return 0;
}

static void *run_thread_retry_to_connect(void *res_in)
{
	struct resources *res = (struct resources *)res_in;
	struct target_details *target;
	time_t sleep_time;

	pthread_mutex_lock(&res->sync_res->retry_mutex);
	while (!res->sync_res->stop_threads) {
		if (retry_list_is_empty(res->sync_res))
			pthread_cond_wait(&res->sync_res->retry_cond,
					  &res->sync_res->retry_mutex);
		while (!res->sync_res->stop_threads &&
		       (target = pop_from_retry_list(res->sync_res)) != NULL) {
			pthread_mutex_unlock(&res->sync_res->retry_mutex);
			sleep_time = target->retry_time - time(NULL);

			if (sleep_time > 0)
				srp_sleep(sleep_time, 0);

			add_non_exist_target(target);
			free(target);
			pthread_mutex_lock(&res->sync_res->retry_mutex);
		}
	}
	/* empty retry_list */
	while ((target = pop_from_retry_list(res->sync_res)))
		free(target);
	pthread_mutex_unlock(&res->sync_res->retry_mutex);

	pr_debug("retry_to_connect thread ended\n");

	pthread_exit(NULL);
}

static void free_res(struct resources *res)
{
	void *status;

	if (!res)
		return;

	if (res->sync_res) {
		pthread_mutex_lock(&res->sync_res->retry_mutex);
		res->sync_res->stop_threads = 1;
		pthread_cond_signal(&res->sync_res->retry_cond);
		pthread_mutex_unlock(&res->sync_res->retry_mutex);
	}

	if (res->ud_res)
		modify_qp_to_err(res->ud_res->qp);

	if (res->reconnect_thread) {
		pthread_kill(res->reconnect_thread, SIGINT);
		pthread_join(res->reconnect_thread, &status);
	}
	if (res->async_ev_thread) {
		pthread_kill(res->async_ev_thread, SIGINT);
		pthread_join(res->async_ev_thread, &status);
	}
	if (res->trap_thread) {
		pthread_kill(res->trap_thread, SIGINT);
		pthread_join(res->trap_thread, &status);
	}
	if (res->sync_res)
		sync_resources_cleanup(res->sync_res);
	if (res->ud_res)
		ud_resources_destroy(res->ud_res);
	if (res->umad_res)
		umad_resources_destroy(res->umad_res);
	free(res);
}

static struct resources *alloc_res(void)
{
	struct all_resources {
		struct resources	res;
		struct ud_resources	ud_res;
		struct umad_resources	umad_res;
		struct sync_resources	sync_res;
	};

	struct all_resources *res;
	int ret;

	res = calloc(1, sizeof(*res));
	if (!res)
		goto err;

	umad_resources_init(&res->umad_res);
	ret = umad_resources_create(&res->umad_res);
	if (ret)
		goto err;
	res->res.umad_res = &res->umad_res;

	ud_resources_init(&res->ud_res);
	ret = ud_resources_create(&res->ud_res);
	if (ret)
		goto err;
	res->res.ud_res = &res->ud_res;
	res->umad_res.ib_ctx = res->ud_res.ib_ctx;

	ret = sync_resources_init(&res->sync_res);
	if (ret)
		goto err;
	res->res.sync_res = &res->sync_res;

	if (!config->once) {
		ret = pthread_create(&res->res.trap_thread, NULL,
				     run_thread_get_trap_notices, &res->res);
		if (ret)
			goto err;

		ret = pthread_create(&res->res.async_ev_thread, NULL,
				     run_thread_listen_to_events, &res->res);
		if (ret)
			goto err;
	}

	if (config->retry_timeout && !config->once) {
		ret = pthread_create(&res->res.reconnect_thread, NULL,
				     run_thread_retry_to_connect, &res->res);
		if (ret)
			goto err;
	}

	return &res->res;
err:
	if (res)
		free_res(&res->res);
	return NULL;
}

/* *c = *a - *b. See also the BSD macro timersub(). */
static void ts_sub(const struct timespec *a, const struct timespec *b,
		   struct timespec *res)
{
	res->tv_sec = a->tv_sec - b->tv_sec;
	res->tv_nsec = a->tv_nsec - b->tv_nsec;
	if (res->tv_nsec < 0) {
		res->tv_sec--;
		res->tv_nsec += 1000 * 1000 * 1000;
	}
}

static void cleanup_wakeup_fd(void)
{
	struct sigaction sa = {};

	sigemptyset(&sa.sa_mask);
	sa.sa_handler = SIG_DFL;
	sigaction(SIGINT, &sa, NULL);
	sigaction(SIGTERM, &sa, NULL);
	sigaction(SRP_CATAS_ERR, &sa, NULL);

	close(wakeup_pipe[1]);
	close(wakeup_pipe[0]);
	wakeup_pipe[0] = -1;
	wakeup_pipe[1] = -1;
}

static int setup_wakeup_fd(void)
{
	struct sigaction sa = {};
	int ret;

	ret = pipe2(wakeup_pipe, O_NONBLOCK | O_CLOEXEC);
	if (ret < 0) {
		pr_err("could not create pipe\n");
		return -1;
	}

	sigemptyset(&sa.sa_mask);
	sa.sa_handler = signal_handler;
	sigaction(SIGINT, &sa, NULL);
	sigaction(SIGTERM, &sa, NULL);
	sigaction(SRP_CATAS_ERR, &sa, NULL);
	return 0;
}

static int ibsrpdm(int argc, char *argv[])
{
	char* umad_dev = NULL;
	struct resources *res;
	int ret;

	s_log_dest = log_to_stderr;

	config = calloc(1, sizeof(*config));
	config->num_of_oust = 10;
	config->timeout = 5000;
	config->mad_retries = 3;
	config->all = 1;
	config->once = 1;

	while (1) {
		int c;

		c = getopt(argc, argv, "cd:h:v");
		if (c == -1)
			break;

		switch (c) {
		case 'c':
			++config->cmd;
			break;
		case 'd':
			umad_dev = optarg;
			break;
		case 'v':
			++config->debug_verbose;
			break;
		case 'h':
		default:
			fprintf(stderr,
				"Usage: %s [-vc] [-d <umad device>]\n",
				argv[0]);
			return 1;
		}
	}

	initialize_sysfs();

	ret = set_conf_dev_and_port(umad_dev, config);
	if (ret) {
		pr_err("Failed to build config\n");
		goto out;
	}

	ret = umad_init();
	if (ret != 0)
		goto out;

	res = alloc_res();
	if (!res) {
		ret = 1;
		pr_err("Resource allocation failed\n");
		goto umad_done;
	}
	ret = recalc(res);
	if (ret)
		pr_err("Querying SRP targets failed\n");

	free_res(res);
umad_done:
	umad_done();
out:
	free_config(config);

	return ret;
}

int main(int argc, char *argv[])
{
	int			ret;
	struct resources       *res;
	uint16_t                lid, sm_lid;
	uint16_t 		pkey;
	union umad_gid 		gid;
	struct target_details  *target;
	int			subscribed;
	int			lockfd = -1;
	int			received_signal = 0;
	bool                    systemd;

#ifndef __CHECKER__
	/*
	 * Hide these checks for sparse because these checks fail with
	 * older versions of sparse.
	 */
	BUILD_ASSERT(sizeof(struct ib_path_rec) == 64);
	BUILD_ASSERT(sizeof(struct ib_inform_info) == 36);
	BUILD_ASSERT(sizeof(struct ib_mad_notice_attr) == 80);
	BUILD_ASSERT(offsetof(struct ib_mad_notice_attr, generic.trap_num) ==
		     4);
	BUILD_ASSERT(offsetof(struct ib_mad_notice_attr, vend.dev_id) == 4);
	BUILD_ASSERT(offsetof(struct ib_mad_notice_attr, ntc_64_67.gid) == 16);
	BUILD_ASSERT(offsetof(struct ib_mad_notice_attr,
			      ntc_144.new_cap_mask) == 16);
#endif
	BUILD_ASSERT(sizeof(struct srp_sa_node_rec) == 108);
	BUILD_ASSERT(sizeof(struct srp_sa_port_info_rec) == 58);
	BUILD_ASSERT(sizeof(struct srp_dm_iou_info) == 132);
	BUILD_ASSERT(sizeof(struct srp_dm_ioc_prof) == 128);

	if (strcmp(argv[0] + max_t(int, 0, strlen(argv[0]) - strlen("ibsrpdm")),
		   "ibsrpdm") == 0) {
		ret = ibsrpdm(argc, argv);
		goto out;
	}

	systemd = is_systemd(argc, argv);

	if (systemd)
		openlog(NULL, LOG_NDELAY | LOG_CONS | LOG_PID, LOG_DAEMON);
	else
		openlog("srp_daemon", LOG_PID, LOG_DAEMON);

	config = calloc(1, sizeof(*config));
	if (!config) {
 		pr_err("out of memory\n");
		ret = ENOMEM;
		goto close_log;
	}

	if (get_config(config, argc, argv)) {
		ret = EINVAL;
		goto free_config;
	}

	if (config->verbose)
		print_config(config);

	if (!config->once) {
		lockfd = check_process_uniqueness(config);
		if (lockfd < 0) {
			ret = EPERM;
			goto free_config;
		}
	}

	ret = setup_wakeup_fd();
	if (ret)
		goto cleanup_wakeup;

catas_start:
	subscribed = 0;

	ret = umad_init();
	if (ret < 0) {
		pr_err("umad_init failed\n");
		goto close_lockfd;
	}

	res = alloc_res();
	if (!res && received_signal == SRP_CATAS_ERR)
		pr_err("Device has not yet recovered from catas error\n");
	if (!res)
		goto clean_umad;

	/*
	 * alloc_res() fails while the HCA is recovering from a catastrophic
	 * error. Clear 'received_signal' after alloc_res() has succeeded to
	 * finish the alloc_res() retry loop.
	 */
	if (received_signal == SRP_CATAS_ERR) {
		pr_err("Device recovered from catastrophic error\n");
		received_signal = 0;
	}

	if (config->once) {
		ret = recalc(res);
		goto free_res;
	}

	while (received_signal == 0) {
		pthread_mutex_lock(&res->sync_res->mutex);
		if (__rescan_scheduled(res->sync_res)) {
			uint16_t port_lid;

			pthread_mutex_unlock(&res->sync_res->mutex);

			pr_debug("Starting a recalculation\n");
			port_lid = get_port_lid(res->ud_res->ib_ctx,
						config->port_num, &sm_lid);
			if (port_lid > 0 && port_lid < 0xc000 &&
			    (port_lid != res->ud_res->port_attr.lid ||
			     sm_lid != res->ud_res->port_attr.sm_lid)) {

				if (res->ud_res->ah) {
					ibv_destroy_ah(res->ud_res->ah);
					res->ud_res->ah = NULL;
				}
				ret = create_ah(res->ud_res);
				if (ret) {
					received_signal = get_received_signal(10, 0);
					goto kill_threads;
				}
			}

			if (res->ud_res->ah) {
				if (register_to_traps(res, 1))
					pr_err("Fail to register to traps, maybe there "
					       "is no SM running on fabric or IB port is down\n");
				else
					subscribed = 1;
			}

			clear_traps_list(res->sync_res);
			schedule_rescan(res->sync_res, config->recalc_time ?
					config->recalc_time : -1);

			/* empty retry_list */
			pthread_mutex_lock(&res->sync_res->retry_mutex);
			while ((target = pop_from_retry_list(res->sync_res)))
				free(target);
			pthread_mutex_unlock(&res->sync_res->retry_mutex);

			recalc(res);
		} else if (pop_from_list(res->sync_res, &lid, &gid, &pkey)) {
			pthread_mutex_unlock(&res->sync_res->mutex);
			if (lid) {
				uint64_t guid;
				ret = get_node(res->umad_res, lid, &guid);
				if (ret)
					/* unexpected error - do a full rescan */
					schedule_rescan(res->sync_res, 0);
				else
					handle_port(res, pkey, lid, guid);
			} else {
				ret = get_lid(res->umad_res, &gid, &lid);
				if (ret < 0)
					/* unexpected error - do a full rescan */
					schedule_rescan(res->sync_res, 0);
				else {
					pr_debug("lid is %#x\n", lid);

					srp_sleep(0, 100);
					handle_port(res, pkey, lid,
						    be64toh(ib_gid_get_guid(&gid)));
				}
			}
		} else {
			static const struct timespec zero;
			struct timespec now, delta;
			struct timespec recalc = {
				.tv_sec = config->recalc_time
			};
			struct timeval timeout;

			clock_gettime(CLOCK_MONOTONIC, &now);
			ts_sub(&res->sync_res->next_recalc_time, &now, &delta);
			pthread_mutex_unlock(&res->sync_res->mutex);

			if (ts_cmp(&zero, &delta, <=) &&
			    ts_cmp(&delta, &recalc, <))
				recalc = delta;
			timeout.tv_sec = recalc.tv_sec;
			timeout.tv_usec = recalc.tv_nsec / 1000 + 1;

			received_signal = get_received_signal(timeout.tv_sec,
							timeout.tv_usec) ? :
				received_signal;
		}
	}

	ret = 0;

kill_threads:
	switch (received_signal) {
	case SIGINT:
		pr_err("Got SIGINT\n");
		break;
	case SIGTERM:
		pr_err("Got SIGTERM\n");
		break;
	case SRP_CATAS_ERR:
		pr_err("Got SIG SRP_CATAS_ERR\n");
		break;
	case 0:
		break;
	default:
		pr_err("Got SIG???\n");
		break;
	}

	if (subscribed && received_signal != SRP_CATAS_ERR) {
		pr_err("Deregistering traps ...\n");
		register_to_traps(res, 0);
		pr_err("Finished trap deregistration.\n");
	}
free_res:
	free_res(res);
	/* Discard the SIGINT triggered by the free_res() implementation. */
	get_received_signal(0, 0);
clean_umad:
	umad_done();
	if (received_signal == SRP_CATAS_ERR) {
		/*
		 * Device got a catastrophic error. Let's wait a grace
		 * period and try to probe the device by attempting to
		 * allocate IB resources. Once it recovers, we will
		 * start all over again.
		 */
		received_signal = get_received_signal(10, 0) ? :
			received_signal;
		if (received_signal == SRP_CATAS_ERR)
			goto catas_start;
	}
close_lockfd:
	if (lockfd >= 0)
		close(lockfd);
cleanup_wakeup:
	cleanup_wakeup_fd();
free_config:
	free_config(config);
close_log:
	closelog();
out:
	exit(ret ? 1 : 0);
}

static int recalc(struct resources *res)
{
	struct umad_resources *umad_res = res->umad_res;
	int  mask_match;
	char val[7];
	int ret;

	ret = srpd_sys_read_string(umad_res->port_sysfs_path, "sm_lid", val, sizeof val);
	if (ret < 0) {
		pr_err("Couldn't read SM LID\n");
		return ret;
	}

	umad_res->sm_lid = strtol(val, NULL, 0);
	if (umad_res->sm_lid == 0) {
		pr_err("SM LID is 0, maybe no SM is running\n");
		return -1;
	}

	ret = check_sm_cap(umad_res, &mask_match);
	if (ret < 0)
		return ret;

	if (mask_match) {
		pr_debug("Advanced SM, performing a capability query\n");
		ret = do_dm_port_list(res);
	} else {
		pr_debug("Old SM, performing a full node query\n");
		ret = do_full_port_list(res);
	}

	return ret;
}

static int get_lid(struct umad_resources *umad_res, union umad_gid *gid,
		   uint16_t *lid)
{
	struct srp_ib_user_mad		out_mad, in_mad;
	struct umad_sa_packet		*in_sa_mad  = get_data_ptr(in_mad);
	struct umad_sa_packet		*out_sa_mad = get_data_ptr(out_mad);
	struct ib_path_rec		*path_rec   = (struct ib_path_rec *) out_sa_mad->data;

	memset(&in_mad, 0, sizeof(in_mad));
	init_srp_sa_mad(&out_mad, umad_res->agent, umad_res->sm_lid,
		        UMAD_SA_ATTR_PATH_REC, 0);

	out_sa_mad->comp_mask = htobe64( 4 | 8 | 64 | 512 | 4096 );

	path_rec->sgid = *gid;
	path_rec->dgid = *gid;
	path_rec->reversible_numpath = 1;
	path_rec->hop_flow_raw = htobe32(1 << 31); /* rawtraffic=1 hoplimit = 0 */

	if (send_and_get(umad_res->portid, umad_res->agent, &out_mad, &in_mad, 0) < 0)
		return -1;

	path_rec = (struct ib_path_rec *) in_sa_mad->data;

	*lid = be16toh(path_rec->dlid);

	return 0;
}