providers/ipathverbs/verbs.c

/*
 * Copyright (c) 2006-2009 QLogic Corp. All rights reserved.
 * Copyright (c) 2005. PathScale, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 * Patent licenses, if any, provided herein do not apply to
 * combinations of this program with other software, or any other
 * product whatsoever.
 */

#include <config.h>

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
#include <sys/mman.h>
#include <errno.h>

#include "ipathverbs.h"
#include "ipath-abi.h"

int ipath_query_device(struct ibv_context *context,
		       const struct ibv_query_device_ex_input *input,
		       struct ibv_device_attr_ex *attr, size_t attr_size)
{
	struct ib_uverbs_ex_query_device_resp resp;
	size_t resp_size = sizeof(resp);
	uint64_t raw_fw_ver;
	unsigned major, minor, sub_minor;
	int ret;

	ret = ibv_cmd_query_device_any(context, input, attr, attr_size, &resp,
				       &resp_size);
	if (ret)
		return ret;

	raw_fw_ver = resp.base.fw_ver;
	major     = (raw_fw_ver >> 32) & 0xffff;
	minor     = (raw_fw_ver >> 16) & 0xffff;
	sub_minor = raw_fw_ver & 0xffff;

	snprintf(attr->orig_attr.fw_ver, sizeof(attr->orig_attr.fw_ver),
		 "%d.%d.%d", major, minor, sub_minor);

	return 0;
}

int ipath_query_port(struct ibv_context *context, uint8_t port,
		     struct ibv_port_attr *attr)
{
	struct ibv_query_port cmd;

	return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd);
}

struct ibv_pd *ipath_alloc_pd(struct ibv_context *context)
{
	struct ibv_alloc_pd	  cmd;
	struct ib_uverbs_alloc_pd_resp  resp;
	struct ibv_pd		 *pd;

	pd = malloc(sizeof *pd);
	if (!pd)
		return NULL;

	if (ibv_cmd_alloc_pd(context, pd, &cmd, sizeof cmd,
			     &resp, sizeof resp)) {
		free(pd);
		return NULL;
	}

	return pd;
}

int ipath_free_pd(struct ibv_pd *pd)
{
	int ret;

	ret = ibv_cmd_dealloc_pd(pd);
	if (ret)
		return ret;

	free(pd);
	return 0;
}

struct ibv_mr *ipath_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
			    uint64_t hca_va, int access)
{
	struct verbs_mr *vmr;
	struct ibv_reg_mr cmd;
	struct ib_uverbs_reg_mr_resp resp;
	int ret;

	vmr = malloc(sizeof(*vmr));
	if (!vmr)
		return NULL;

	ret = ibv_cmd_reg_mr(pd, addr, length, hca_va, access, vmr, &cmd,
			     sizeof(cmd), &resp, sizeof(resp));
	if (ret) {
		free(vmr);
		return NULL;
	}

	return &vmr->ibv_mr;
}

int ipath_dereg_mr(struct verbs_mr *vmr)
{
	int ret;

	ret = ibv_cmd_dereg_mr(vmr);
	if (ret)
		return ret;

	free(vmr);
	return 0;
}

struct ibv_cq *ipath_create_cq(struct ibv_context *context, int cqe,
			       struct ibv_comp_channel *channel,
			       int comp_vector)
{
	struct ipath_cq		   *cq;
	struct ipath_create_cq_resp resp;
	int			    ret;
	size_t			    size;

	cq = malloc(sizeof *cq);
	if (!cq)
		return NULL;

	ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector,
				&cq->ibv_cq, NULL, 0,
				&resp.ibv_resp, sizeof resp);
	if (ret) {
		free(cq);
		return NULL;
	}

	size = sizeof(struct ipath_cq_wc) + sizeof(struct ipath_wc) * cqe;
	cq->queue = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED,
			 context->cmd_fd, resp.offset);
	if ((void *) cq->queue == MAP_FAILED) {
		ibv_cmd_destroy_cq(&cq->ibv_cq);
		free(cq);
		return NULL;
	}

	pthread_spin_init(&cq->lock, PTHREAD_PROCESS_PRIVATE);
	return &cq->ibv_cq;
}

struct ibv_cq *ipath_create_cq_v1(struct ibv_context *context, int cqe,
				  struct ibv_comp_channel *channel,
				  int comp_vector)
{
	struct ibv_cq		   *cq;
	int			    ret;

	cq = malloc(sizeof *cq);
	if (!cq)
		return NULL;

	ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector,
				cq, NULL, 0, NULL, 0);
	if (ret) {
		free(cq);
		return NULL;
	}

	return cq;
}

int ipath_resize_cq(struct ibv_cq *ibcq, int cqe)
{
	struct ipath_cq		       *cq = to_icq(ibcq);
	struct ibv_resize_cq		cmd;
	struct ipath_resize_cq_resp	resp;
	size_t				size;
	int				ret;

	pthread_spin_lock(&cq->lock);
	/* Save the old size so we can unmmap the queue. */
	size = sizeof(struct ipath_cq_wc) +
		(sizeof(struct ipath_wc) * cq->ibv_cq.cqe);
	ret = ibv_cmd_resize_cq(ibcq, cqe, &cmd, sizeof cmd,
				&resp.ibv_resp, sizeof resp);
	if (ret) {
		pthread_spin_unlock(&cq->lock);
		return ret;
	}
	(void) munmap(cq->queue, size);
	size = sizeof(struct ipath_cq_wc) +
		(sizeof(struct ipath_wc) * cq->ibv_cq.cqe);
	cq->queue = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED,
			 ibcq->context->cmd_fd, resp.offset);
	ret = errno;
	pthread_spin_unlock(&cq->lock);
	if ((void *) cq->queue == MAP_FAILED)
		return ret;
	return 0;
}

int ipath_resize_cq_v1(struct ibv_cq *ibcq, int cqe)
{
	struct ibv_resize_cq		cmd;
	struct ib_uverbs_resize_cq_resp	resp;

	return ibv_cmd_resize_cq(ibcq, cqe, &cmd, sizeof cmd,
				 &resp, sizeof resp);
}

int ipath_destroy_cq(struct ibv_cq *ibcq)
{
	struct ipath_cq *cq = to_icq(ibcq);
	int ret;

	ret = ibv_cmd_destroy_cq(ibcq);
	if (ret)
		return ret;

	(void) munmap(cq->queue, sizeof(struct ipath_cq_wc) +
				 (sizeof(struct ipath_wc) * cq->ibv_cq.cqe));
	free(cq);
	return 0;
}

int ipath_destroy_cq_v1(struct ibv_cq *ibcq)
{
	int ret;

	ret = ibv_cmd_destroy_cq(ibcq);
	if (!ret)
		free(ibcq);
	return ret;
}

int ipath_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc)
{
	struct ipath_cq *cq = to_icq(ibcq);
	struct ipath_cq_wc *q;
	int npolled;
	uint32_t tail;

	pthread_spin_lock(&cq->lock);
	q = cq->queue;
	tail = atomic_load_explicit(&q->tail, memory_order_relaxed);
	for (npolled = 0; npolled < ne; ++npolled, ++wc) {
		if (tail == atomic_load(&q->head))
			break;

		/* Make sure entry is read after head index is read. */
		atomic_thread_fence(memory_order_acquire);
		memcpy(wc, &q->queue[tail], sizeof(*wc));
		if (tail == cq->ibv_cq.cqe)
			tail = 0;
		else
			tail++;
	}
	atomic_store(&q->tail, tail);
	pthread_spin_unlock(&cq->lock);

	return npolled;
}

struct ibv_qp *ipath_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
{
	struct ibv_create_qp	     cmd;
	struct ipath_create_qp_resp  resp;
	struct ipath_qp		    *qp;
	int			     ret;
	size_t			     size;

	qp = malloc(sizeof *qp);
	if (!qp)
		return NULL;

	ret = ibv_cmd_create_qp(pd, &qp->ibv_qp, attr, &cmd, sizeof cmd,
				&resp.ibv_resp, sizeof resp);
	if (ret) {
		free(qp);
		return NULL;
	}

	if (attr->srq) {
		qp->rq.size = 0;
		qp->rq.max_sge = 0;
		qp->rq.rwq = NULL;
	} else {
		qp->rq.size = attr->cap.max_recv_wr + 1;
		qp->rq.max_sge = attr->cap.max_recv_sge;
		size = sizeof(struct ipath_rwq) +
			(sizeof(struct ipath_rwqe) +
			 (sizeof(struct ibv_sge) * qp->rq.max_sge)) *
			qp->rq.size;
		qp->rq.rwq = mmap(NULL, size,
				  PROT_READ | PROT_WRITE, MAP_SHARED,
				  pd->context->cmd_fd, resp.offset);
		if ((void *) qp->rq.rwq == MAP_FAILED) {
			ibv_cmd_destroy_qp(&qp->ibv_qp);
			free(qp);
			return NULL;
		}
	}

	pthread_spin_init(&qp->rq.lock, PTHREAD_PROCESS_PRIVATE);
	return &qp->ibv_qp;
}

struct ibv_qp *ipath_create_qp_v1(struct ibv_pd *pd,
				  struct ibv_qp_init_attr *attr)
{
	struct ibv_create_qp	     cmd;
	struct ib_uverbs_create_qp_resp    resp;
	struct ibv_qp		    *qp;
	int			     ret;

	qp = malloc(sizeof *qp);
	if (!qp)
		return NULL;

	ret = ibv_cmd_create_qp(pd, qp, attr, &cmd, sizeof cmd,
				&resp, sizeof resp);
	if (ret) {
		free(qp);
		return NULL;
	}

	return qp;
}

int ipath_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
		   int attr_mask,
		   struct ibv_qp_init_attr *init_attr)
{
	struct ibv_query_qp cmd;

	return ibv_cmd_query_qp(qp, attr, attr_mask, init_attr,
				&cmd, sizeof cmd);
}

int ipath_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
		    int attr_mask)
{
	struct ibv_modify_qp cmd = {};

	return ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof cmd);
}

int ipath_destroy_qp(struct ibv_qp *ibqp)
{
	struct ipath_qp	*qp = to_iqp(ibqp);
	int ret;

	ret = ibv_cmd_destroy_qp(ibqp);
	if (ret)
		return ret;

	if (qp->rq.rwq) {
		size_t size;

		size = sizeof(struct ipath_rwq) +
			(sizeof(struct ipath_rwqe) +
			 (sizeof(struct ibv_sge) * qp->rq.max_sge)) *
			qp->rq.size;
		(void) munmap(qp->rq.rwq, size);
	}
	free(qp);
	return 0;
}

int ipath_destroy_qp_v1(struct ibv_qp *ibqp)
{
	int ret;

	ret = ibv_cmd_destroy_qp(ibqp);
	if (!ret)
		free(ibqp);
	return ret;
}

int ipath_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr,
		    struct ibv_send_wr **bad_wr)
{
	unsigned wr_count;
	struct ibv_send_wr *i;

	/* Sanity check the number of WRs being posted */
	for (i = wr, wr_count = 0; i; i = i->next)
		if (++wr_count > 10)
			goto iter;

	return ibv_cmd_post_send(qp, wr, bad_wr);

iter:
	do {
		struct ibv_send_wr *next;
		int ret;

		next = i->next;
		i->next = NULL;
		ret = ibv_cmd_post_send(qp, wr, bad_wr);
		i->next = next;
		if (ret)
			return ret;
		if (next == NULL)
			break;
		wr = next;
		for (i = wr, wr_count = 0; i->next; i = i->next)
			if (++wr_count > 2)
				break;
	} while (1);
	return 0;
}

static int post_recv(struct ipath_rq *rq, struct ibv_recv_wr *wr,
		     struct ibv_recv_wr **bad_wr)
{
	struct ibv_recv_wr *i;
	struct ipath_rwq *rwq;
	struct ipath_rwqe *wqe;
	uint32_t head;
	int n, ret;

	pthread_spin_lock(&rq->lock);
	rwq = rq->rwq;
	head = atomic_load_explicit(&rwq->head, memory_order_relaxed);
	for (i = wr; i; i = i->next) {
		if ((unsigned) i->num_sge > rq->max_sge) {
			ret = EINVAL;
			goto bad;
		}
		wqe = get_rwqe_ptr(rq, head);
		if (++head >= rq->size)
			head = 0;
		if (head == atomic_load(&rwq->tail)) {
			ret = ENOMEM;
			goto bad;
		}
		wqe->wr_id = i->wr_id;
		wqe->num_sge = i->num_sge;
		for (n = 0; n < wqe->num_sge; n++)
			wqe->sg_list[n] = i->sg_list[n];

		/* Make sure queue entry is written before the head index. */
		atomic_thread_fence(memory_order_release);
		atomic_store(&rwq->head, head);
	}
	ret = 0;
	goto done;

bad:
	if (bad_wr)
		*bad_wr = i;
done:
	pthread_spin_unlock(&rq->lock);
	return ret;
}

int ipath_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
		    struct ibv_recv_wr **bad_wr)
{
	struct ipath_qp *qp = to_iqp(ibqp);

	return post_recv(&qp->rq, wr, bad_wr);
}

struct ibv_srq *ipath_create_srq(struct ibv_pd *pd,
				 struct ibv_srq_init_attr *attr)
{
	struct ipath_srq *srq;
	struct ibv_create_srq cmd;
	struct ipath_create_srq_resp resp;
	int ret;
	size_t size;

	srq = malloc(sizeof *srq);
	if (srq == NULL)
		return NULL;

	ret = ibv_cmd_create_srq(pd, &srq->ibv_srq, attr, &cmd, sizeof cmd,
				 &resp.ibv_resp, sizeof resp);
	if (ret) {
		free(srq);
		return NULL;
	}

	srq->rq.size = attr->attr.max_wr + 1;
	srq->rq.max_sge = attr->attr.max_sge;
	size = sizeof(struct ipath_rwq) +
		(sizeof(struct ipath_rwqe) +
		 (sizeof(struct ibv_sge) * srq->rq.max_sge)) * srq->rq.size;
	srq->rq.rwq = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED,
			   pd->context->cmd_fd, resp.offset);
	if ((void *) srq->rq.rwq == MAP_FAILED) {
		ibv_cmd_destroy_srq(&srq->ibv_srq);
		free(srq);
		return NULL;
	}

	pthread_spin_init(&srq->rq.lock, PTHREAD_PROCESS_PRIVATE);
	return &srq->ibv_srq;
}

struct ibv_srq *ipath_create_srq_v1(struct ibv_pd *pd,
				    struct ibv_srq_init_attr *attr)
{
	struct ibv_srq *srq;
	struct ibv_create_srq cmd;
	struct ib_uverbs_create_srq_resp resp;
	int ret;

	srq = malloc(sizeof *srq);
	if (srq == NULL)
		return NULL;

	ret = ibv_cmd_create_srq(pd, srq, attr, &cmd, sizeof cmd,
				 &resp, sizeof resp);
	if (ret) {
		free(srq);
		return NULL;
	}

	return srq;
}

int ipath_modify_srq(struct ibv_srq *ibsrq,
		     struct ibv_srq_attr *attr, 
		     int attr_mask)
{
	struct ipath_srq            *srq = to_isrq(ibsrq);
	struct ipath_modify_srq_cmd  cmd;
	__u64                        offset;
	size_t                       size = 0; /* Shut up gcc */
	int                          ret;

	if (attr_mask & IBV_SRQ_MAX_WR) {
		pthread_spin_lock(&srq->rq.lock);
		/* Save the old size so we can unmmap the queue. */
		size = sizeof(struct ipath_rwq) +
			(sizeof(struct ipath_rwqe) +
			 (sizeof(struct ibv_sge) * srq->rq.max_sge)) *
			srq->rq.size;
	}
	cmd.offset_addr = (uintptr_t) &offset;
	ret = ibv_cmd_modify_srq(ibsrq, attr, attr_mask,
				 &cmd.ibv_cmd, sizeof cmd);
	if (ret) {
		if (attr_mask & IBV_SRQ_MAX_WR)
			pthread_spin_unlock(&srq->rq.lock);
		return ret;
	}
	if (attr_mask & IBV_SRQ_MAX_WR) {
		(void) munmap(srq->rq.rwq, size);
		srq->rq.size = attr->max_wr + 1;
		size = sizeof(struct ipath_rwq) +
			(sizeof(struct ipath_rwqe) +
			 (sizeof(struct ibv_sge) * srq->rq.max_sge)) *
			srq->rq.size;
		srq->rq.rwq = mmap(NULL, size,
				   PROT_READ | PROT_WRITE, MAP_SHARED,
				   ibsrq->context->cmd_fd, offset);
		pthread_spin_unlock(&srq->rq.lock);
		/* XXX Now we have no receive queue. */
		if ((void *) srq->rq.rwq == MAP_FAILED)
			return errno;
	}
	return 0;
}

int ipath_modify_srq_v1(struct ibv_srq *ibsrq,
			struct ibv_srq_attr *attr, 
			int attr_mask)
{
	struct ibv_modify_srq cmd;

	return ibv_cmd_modify_srq(ibsrq, attr, attr_mask,
				  &cmd, sizeof cmd);
}

int ipath_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr)
{
	struct ibv_query_srq cmd;

	return ibv_cmd_query_srq(srq, attr, &cmd, sizeof cmd);
}

int ipath_destroy_srq(struct ibv_srq *ibsrq)
{
	struct ipath_srq *srq = to_isrq(ibsrq);
	size_t size;
	int ret;

	ret = ibv_cmd_destroy_srq(ibsrq);
	if (ret)
		return ret;

	size = sizeof(struct ipath_rwq) +
		(sizeof(struct ipath_rwqe) +
		 (sizeof(struct ibv_sge) * srq->rq.max_sge)) * srq->rq.size;
	(void) munmap(srq->rq.rwq, size);
	free(srq);
	return 0;
}

int ipath_destroy_srq_v1(struct ibv_srq *ibsrq)
{
	int ret;

	ret = ibv_cmd_destroy_srq(ibsrq);
	if (!ret)
		free(ibsrq);
	return ret;
}

int ipath_post_srq_recv(struct ibv_srq *ibsrq, struct ibv_recv_wr *wr,
			struct ibv_recv_wr **bad_wr)
{
	struct ipath_srq *srq = to_isrq(ibsrq);

	return post_recv(&srq->rq, wr, bad_wr); 
}

struct ibv_ah *ipath_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
{
	struct ibv_ah *ah;
	struct ib_uverbs_create_ah_resp resp;

	ah = malloc(sizeof *ah);
	if (ah == NULL)
		return NULL;

	memset(&resp, 0, sizeof(resp));
	if (ibv_cmd_create_ah(pd, ah, attr, &resp, sizeof(resp))) {
		free(ah);
		return NULL;
	}

	return ah;
}

int ipath_destroy_ah(struct ibv_ah *ah)
{
	int ret;

	ret = ibv_cmd_destroy_ah(ah);
	if (ret)
		return ret;

	free(ah);
	return 0;
}