providers/ocrdma/ocrdma_verbs.c

/*
 * Copyright (C) 2008-2013 Emulex.  All rights reserved.
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT  LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR  A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <config.h>

#include <assert.h>
#include <endian.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <signal.h>
#include <errno.h>
#include <pthread.h>
#include <sys/mman.h>
#include <unistd.h>
#include <endian.h>

#include "ocrdma_main.h"
#include "ocrdma_abi.h"
#include <ccan/list.h>
#include <util/compiler.h>

static void ocrdma_ring_cq_db(struct ocrdma_cq *cq, uint32_t armed,
			      int solicited, uint32_t num_cqe);

static inline void ocrdma_swap_cpu_to_le(void *dst, uint32_t len)
{
	int i = 0;
        __le32 *src_ptr = dst;
	uint32_t *dst_ptr = dst;
	for (; i < (len / 4); i++)
		*dst_ptr++ = le32toh(*src_ptr++);
}

/*
 * ocrdma_query_device
 */
int ocrdma_query_device(struct ibv_context *context,
			const struct ibv_query_device_ex_input *input,
			struct ibv_device_attr_ex *attr, size_t attr_size)
{
	struct ocrdma_device *dev = get_ocrdma_dev(context->device);
	struct ib_uverbs_ex_query_device_resp resp;
	size_t resp_size = sizeof(resp);
	int ret;

	ret = ibv_cmd_query_device_any(context, input, attr, attr_size, &resp,
				       &resp_size);
	if (ret)
		return ret;

	memcpy(attr->orig_attr.fw_ver, dev->fw_ver, sizeof(dev->fw_ver));
	return 0;
}

/*
 * ocrdma_query_port
 */
int ocrdma_query_port(struct ibv_context *context, uint8_t port,
		      struct ibv_port_attr *attr)
{
	struct ibv_query_port cmd;
	int status;
	status = ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd);
	return status;
}

#define OCRDMA_INVALID_AH_IDX 0xffffffff
void ocrdma_init_ahid_tbl(struct ocrdma_devctx *ctx)
{
	int i;

	pthread_mutex_init(&ctx->tbl_lock, NULL);
	for (i = 0; i < (ctx->ah_tbl_len / sizeof(uint32_t)); i++)
		ctx->ah_tbl[i] = OCRDMA_INVALID_AH_IDX;
}

static int ocrdma_alloc_ah_tbl_id(struct ocrdma_devctx *ctx)
{
	int i;
	int status = -EINVAL;
	pthread_mutex_lock(&ctx->tbl_lock);

	for (i = 0; i < (ctx->ah_tbl_len / sizeof(uint32_t)); i++) {
		if (ctx->ah_tbl[i] == OCRDMA_INVALID_AH_IDX) {
			ctx->ah_tbl[i] = ctx->ah_tbl_len;
			status = i;
			break;
		}
	}
	pthread_mutex_unlock(&ctx->tbl_lock);
	return status;
}

static void ocrdma_free_ah_tbl_id(struct ocrdma_devctx *ctx, int idx)
{
	pthread_mutex_lock(&ctx->tbl_lock);
	ctx->ah_tbl[idx] = OCRDMA_INVALID_AH_IDX;
	pthread_mutex_unlock(&ctx->tbl_lock);
}

/*
 * ocrdma_alloc_pd
 */
struct ibv_pd *ocrdma_alloc_pd(struct ibv_context *context)
{
	struct uocrdma_alloc_pd cmd;
	struct uocrdma_alloc_pd_resp resp;
	struct ocrdma_pd *pd;
	uint64_t map_address = 0;

	pd = malloc(sizeof *pd);
	if (!pd)
		return NULL;
	bzero(pd, sizeof *pd);
	memset(&cmd, 0, sizeof(cmd));

	if (ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd.ibv_cmd, sizeof(cmd),
			     &resp.ibv_resp, sizeof(resp))) {
		free(pd);
		return NULL;
	}
	pd->dev = get_ocrdma_dev(context->device);
	pd->uctx = get_ocrdma_ctx(context);

	if (resp.dpp_enabled) {
		map_address = ((uint64_t) resp.dpp_page_addr_hi << 32) |
		    resp.dpp_page_addr_lo;
		pd->dpp_va = mmap(NULL, OCRDMA_DPP_PAGE_SIZE, PROT_WRITE,
				  MAP_SHARED, context->cmd_fd, map_address);
		if (pd->dpp_va == MAP_FAILED) {
			ocrdma_free_pd(&pd->ibv_pd);
			return NULL;
		}
	}
	return &pd->ibv_pd;
}

/*
 * ocrdma_free_pd
 */
int ocrdma_free_pd(struct ibv_pd *ibpd)
{
	int status;
	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);

	status = ibv_cmd_dealloc_pd(ibpd);
	if (status)
		return status;

	if (pd->dpp_va)
		munmap((void *)pd->dpp_va, OCRDMA_DPP_PAGE_SIZE);
	free(pd);
	return 0;
}

/*
 * ocrdma_reg_mr
 */
struct ibv_mr *ocrdma_reg_mr(struct ibv_pd *pd, void *addr, size_t len,
			     uint64_t hca_va, int access)
{
	struct ocrdma_mr *mr;
	struct ibv_reg_mr cmd;
	struct uocrdma_reg_mr_resp resp;

	mr = malloc(sizeof *mr);
	if (!mr)
		return NULL;
	bzero(mr, sizeof *mr);

	if (ibv_cmd_reg_mr(pd, addr, len, hca_va, access, &mr->vmr, &cmd,
			   sizeof(cmd), &resp.ibv_resp, sizeof(resp))) {
		free(mr);
		return NULL;
	}
	return &mr->vmr.ibv_mr;
}

/*
 * ocrdma_dereg_mr
 */
int ocrdma_dereg_mr(struct verbs_mr *vmr)
{
	int status;
	status = ibv_cmd_dereg_mr(vmr);
	if (status)
		return status;
	free(vmr);
	return 0;
}

/*
 * ocrdma_create_cq
 */
static struct ibv_cq *ocrdma_create_cq_common(struct ibv_context *context,
					      int cqe,
					      struct ibv_comp_channel *channel,
					      int comp_vector, int dpp_cq)
{
	int status;
	struct uocrdma_create_cq cmd;
	struct uocrdma_create_cq_resp resp;
	struct ocrdma_cq *cq;
	struct ocrdma_device *dev = get_ocrdma_dev(context->device);
	void *map_addr;

	cq = malloc(sizeof *cq);
	if (!cq)
		return NULL;

	bzero(cq, sizeof *cq);
	cmd.dpp_cq = dpp_cq;
	status = ibv_cmd_create_cq(context, cqe, channel, comp_vector,
				   &cq->ibv_cq, &cmd.ibv_cmd, sizeof cmd,
				   &resp.ibv_resp, sizeof resp);
	if (status)
		goto cq_err1;

	pthread_spin_init(&cq->cq_lock, PTHREAD_PROCESS_PRIVATE);
	cq->dev = dev;
	cq->cq_id = resp.cq_id;
	cq->cq_dbid = resp.cq_id;
	cq->cq_mem_size = resp.page_size;
	cq->max_hw_cqe = resp.max_hw_cqe;
	cq->phase_change = resp.phase_change;
	cq->va = mmap(NULL, resp.page_size, PROT_READ | PROT_WRITE,
		      MAP_SHARED, context->cmd_fd, resp.page_addr[0]);
	if (cq->va == MAP_FAILED)
		goto cq_err2;

	map_addr = mmap(NULL, resp.db_page_size, PROT_WRITE,
			MAP_SHARED, context->cmd_fd, resp.db_page_addr);
	if (map_addr == MAP_FAILED)
		goto cq_err2;
	cq->db_va = map_addr;
	cq->db_size = resp.db_page_size;
	cq->phase = OCRDMA_CQE_VALID;
	cq->first_arm = 1;
	if (!dpp_cq) {
		ocrdma_ring_cq_db(cq, 0, 0, 0);
	}
	cq->ibv_cq.cqe = cqe;
	list_head_init(&cq->sq_head);
	list_head_init(&cq->rq_head);
	return &cq->ibv_cq;
cq_err2:
	(void)ibv_cmd_destroy_cq(&cq->ibv_cq);
cq_err1:
	free(cq);
	return NULL;
}

struct ibv_cq *ocrdma_create_cq(struct ibv_context *context, int cqe,
				struct ibv_comp_channel *channel,
				int comp_vector)
{
	return ocrdma_create_cq_common(context, cqe, channel, comp_vector, 0);
}

#ifdef DPP_CQ_SUPPORT
static struct ocrdma_cq *ocrdma_create_dpp_cq(struct ibv_context *context,
					      int cqe)
{
	struct ibv_cq *ibcq;
	ibcq = ocrdma_create_cq_common(context, cqe, 0, 0, 1);
	if (ibcq)
		return get_ocrdma_cq(ibcq);
	return NULL;
}
#endif

/*
 * ocrdma_resize_cq
 */
int ocrdma_resize_cq(struct ibv_cq *ibcq, int new_entries)
{
	int status;
	struct ibv_resize_cq cmd;
	struct ib_uverbs_resize_cq_resp resp;
	status = ibv_cmd_resize_cq(ibcq, new_entries,
				   &cmd, sizeof cmd, &resp, sizeof resp);
	if (status == 0)
		ibcq->cqe = new_entries;
	return status;
}

/*
 * ocrdma_destroy_cq
 */
int ocrdma_destroy_cq(struct ibv_cq *ibv_cq)
{
	struct ocrdma_cq *cq = get_ocrdma_cq(ibv_cq);
	int status;

	status = ibv_cmd_destroy_cq(ibv_cq);
	if (status)
		return status;

	if (cq->db_va)
		munmap((void *)cq->db_va, cq->db_size);
	if (cq->va)
		munmap((void*)cq->va, cq->cq_mem_size);

	free(cq);
	return 0;
}

static void ocrdma_add_qpn_map(struct ocrdma_device *dev, struct ocrdma_qp *qp)
{
	pthread_mutex_lock(&dev->dev_lock);
	dev->qp_tbl[qp->id] = qp;
	pthread_mutex_unlock(&dev->dev_lock);
}

static void _ocrdma_del_qpn_map(struct ocrdma_device *dev, struct ocrdma_qp *qp)
{
	dev->qp_tbl[qp->id] = NULL;
}

struct ibv_srq *ocrdma_create_srq(struct ibv_pd *pd,
				  struct ibv_srq_init_attr *init_attr)
{
	int status = 0;
	struct ocrdma_srq *srq;
	struct uocrdma_create_srq cmd;
	struct uocrdma_create_srq_resp resp = {};
	void *map_addr;

	srq = calloc(1, sizeof *srq);
	if (!srq)
		return NULL;

	pthread_spin_init(&srq->q_lock, PTHREAD_PROCESS_PRIVATE);
	status = ibv_cmd_create_srq(pd, &srq->ibv_srq, init_attr, &cmd.ibv_cmd,
				    sizeof cmd, &resp.ibv_resp, sizeof resp);
	if (status)
		goto cmd_err;

	srq->dev = get_ocrdma_pd(pd)->dev;
	srq->rq.dbid = resp.rq_dbid;
	srq->rq.max_sges = init_attr->attr.max_sge;
	srq->rq.max_cnt = resp.num_rqe_allocated;
	srq->rq.max_wqe_idx = resp.num_rqe_allocated - 1;
	srq->rq.entry_size = srq->dev->rqe_size;
	srq->rqe_wr_id_tbl = calloc(srq->rq.max_cnt, sizeof(uint64_t));
	if (srq->rqe_wr_id_tbl == NULL)
		goto map_err;

	srq->bit_fields_len =
	    (srq->rq.max_cnt / 32) + (srq->rq.max_cnt % 32 ? 1 : 0);
	srq->idx_bit_fields = malloc(srq->bit_fields_len * sizeof(uint32_t));
	if (srq->idx_bit_fields == NULL)
		goto map_err;
	memset(srq->idx_bit_fields, 0xff,
	       srq->bit_fields_len * sizeof(uint32_t));

	if (resp.num_rq_pages > 1)
		goto map_err;

	map_addr = mmap(NULL, resp.rq_page_size, PROT_READ | PROT_WRITE,
			MAP_SHARED, pd->context->cmd_fd, resp.rq_page_addr[0]);
	if (map_addr == MAP_FAILED)
		goto map_err;
	srq->rq.len = resp.rq_page_size;
	srq->rq.va = map_addr;

	map_addr = mmap(NULL, resp.db_page_size, PROT_WRITE,
			MAP_SHARED, pd->context->cmd_fd, resp.db_page_addr);
	if (map_addr == MAP_FAILED)
		goto map_err;
	srq->db_va = (uint8_t *) map_addr + resp.db_rq_offset;
	srq->db_shift = resp.db_shift;
	srq->db_size = resp.db_page_size;
	return &srq->ibv_srq;

map_err:
	ocrdma_destroy_srq(&srq->ibv_srq);
	return NULL;

cmd_err:
	pthread_spin_destroy(&srq->q_lock);
	free(srq);
	return NULL;
}

int ocrdma_modify_srq(struct ibv_srq *ibsrq,
		      struct ibv_srq_attr *attr, int attr_mask)
{
	struct ibv_modify_srq cmd;

	return ibv_cmd_modify_srq(ibsrq, attr, attr_mask, &cmd, sizeof cmd);
}

int ocrdma_query_srq(struct ibv_srq *ibsrq, struct ibv_srq_attr *attr)
{
	struct ibv_query_srq cmd;

	return ibv_cmd_query_srq(ibsrq, attr, &cmd, sizeof cmd);
}

int ocrdma_destroy_srq(struct ibv_srq *ibsrq)
{
	int status;
	struct ocrdma_srq *srq;
	srq = get_ocrdma_srq(ibsrq);

	status = ibv_cmd_destroy_srq(ibsrq);
	if (status)
		return status;

	if (srq->idx_bit_fields)
		free(srq->idx_bit_fields);
	if (srq->rqe_wr_id_tbl)
		free(srq->rqe_wr_id_tbl);
	if (srq->db_va) {
		munmap((void *)srq->db_va, srq->db_size);
		srq->db_va = NULL;
	}
	if (srq->rq.va) {
		munmap(srq->rq.va, srq->rq.len);
		srq->rq.va = NULL;
	}
	pthread_spin_destroy(&srq->q_lock);
	free(srq);
	return status;
}

/*
 * ocrdma_create_qp
 */
struct ibv_qp *ocrdma_create_qp(struct ibv_pd *pd,
				struct ibv_qp_init_attr *attrs)
{
	int status = 0;
	struct uocrdma_create_qp cmd;
	struct uocrdma_create_qp_resp resp = {};
	struct ocrdma_qp *qp;
	void *map_addr;
#ifdef DPP_CQ_SUPPORT
	struct ocrdma_dpp_cqe *dpp_cqe = NULL;
#endif

	qp = calloc(1, sizeof *qp);
	if (!qp)
		return NULL;
	memset(&cmd, 0, sizeof(cmd));

	qp->qp_type = attrs->qp_type;
	pthread_spin_init(&qp->q_lock, PTHREAD_PROCESS_PRIVATE);

#ifdef DPP_CQ_SUPPORT
	if (attrs->cap.max_inline_data) {
		qp->dpp_cq = ocrdma_create_dpp_cq(pd->context,
					OCRDMA_CREATE_QP_REQ_DPP_CREDIT_LIMIT);
		if (qp->dpp_cq) {
			cmd.enable_dpp_cq = 1;
			cmd.dpp_cq_id = qp->dpp_cq->cq_id;
			/* Write invalid index for the first entry */
			dpp_cqe = (struct ocrdma_dpp_cqe *)qp->dpp_cq->va;
			dpp_cqe->wqe_idx_valid = 0xFFFF;
			qp->dpp_prev_indx = 0xFFFF;
		}
	}
#endif
	status = ibv_cmd_create_qp(pd, &qp->ibv_qp, attrs, &cmd.ibv_cmd,
				   sizeof cmd, &resp.ibv_resp, sizeof resp);
	if (status)
		goto mbx_err;

	qp->dev = get_ocrdma_dev(pd->context->device);
	qp->id = resp.qp_id;

	ocrdma_add_qpn_map(qp->dev, qp);

	qp->sq.dbid = resp.sq_dbid;

	qp->sq.max_sges = attrs->cap.max_send_sge;
	qp->max_inline_data = attrs->cap.max_inline_data;

	qp->signaled = attrs->sq_sig_all;

	qp->sq.max_cnt = resp.num_wqe_allocated;
	qp->sq.max_wqe_idx = resp.num_wqe_allocated - 1;
	qp->sq.entry_size = qp->dev->wqe_size;
	if (attrs->srq)
		qp->srq = get_ocrdma_srq(attrs->srq);
	else {
		qp->rq.dbid = resp.rq_dbid;
		qp->rq.max_sges = attrs->cap.max_recv_sge;
		qp->rq.max_cnt = resp.num_rqe_allocated;
		qp->rq.max_wqe_idx = resp.num_rqe_allocated - 1;
		qp->rq.entry_size = qp->dev->rqe_size;
		qp->rqe_wr_id_tbl = calloc(qp->rq.max_cnt, sizeof(uint64_t));
		if (qp->rqe_wr_id_tbl == NULL)
			goto map_err;
	}

	qp->sq_cq = get_ocrdma_cq(attrs->send_cq);
	qp->rq_cq = get_ocrdma_cq(attrs->recv_cq);

	qp->wqe_wr_id_tbl = calloc(qp->sq.max_cnt, sizeof(*qp->wqe_wr_id_tbl));
	if (qp->wqe_wr_id_tbl == NULL)
		goto map_err;

	/* currently we support only one virtual page */
	if ((resp.num_sq_pages > 1) || (!attrs->srq && resp.num_rq_pages > 1))
		goto map_err;

	map_addr = mmap(NULL, resp.sq_page_size, PROT_READ | PROT_WRITE,
			MAP_SHARED, pd->context->cmd_fd, resp.sq_page_addr[0]);
	if (map_addr == MAP_FAILED)
		goto map_err;
	qp->sq.va = map_addr;
	qp->sq.len = resp.sq_page_size;
	qp->db_shift = resp.db_shift;

	if (!attrs->srq) {
		map_addr = mmap(NULL, resp.rq_page_size, PROT_READ | PROT_WRITE,
				MAP_SHARED, pd->context->cmd_fd,
				resp.rq_page_addr[0]);
		if (map_addr == MAP_FAILED)
			goto map_err;

		qp->rq.len = resp.rq_page_size;
		qp->rq.va = map_addr;
	}

	map_addr = mmap(NULL, resp.db_page_size, PROT_WRITE,
			MAP_SHARED, pd->context->cmd_fd, resp.db_page_addr);
	if (map_addr == MAP_FAILED)
		goto map_err;

	qp->db_va = map_addr;
	qp->db_sq_va = (uint8_t *) map_addr + resp.db_sq_offset;
	qp->db_rq_va = (uint8_t *) map_addr + resp.db_rq_offset;

	qp->db_size = resp.db_page_size;

	if (resp.dpp_credit) {
		struct ocrdma_pd *opd = get_ocrdma_pd(pd);
		map_addr = (uint8_t *) opd->dpp_va +
		    (resp.dpp_offset * qp->dev->wqe_size);
		qp->dpp_q.max_cnt = 1;	/* DPP is posted at the same offset */
		qp->dpp_q.free_cnt = resp.dpp_credit;
		qp->dpp_q.va = map_addr;
		qp->dpp_q.head = qp->dpp_q.tail = 0;
		qp->dpp_q.entry_size = qp->dev->dpp_wqe_size;
		qp->dpp_q.len = resp.dpp_credit * qp->dev->dpp_wqe_size;
		qp->dpp_enabled = 1;
	} else {
		if (qp->dpp_cq) {
			ocrdma_destroy_cq(&qp->dpp_cq->ibv_cq);
			qp->dpp_cq = NULL;
		}
	}
	qp->state = OCRDMA_QPS_RST;
	list_node_init(&qp->sq_entry);
	list_node_init(&qp->rq_entry);
	return &qp->ibv_qp;

map_err:
	ocrdma_destroy_qp(&qp->ibv_qp);
	return NULL;
mbx_err:
	pthread_spin_destroy(&qp->q_lock);
	free(qp);
	return NULL;
}

static enum ocrdma_qp_state get_ocrdma_qp_state(enum ibv_qp_state qps)
{
	switch (qps) {
	case IBV_QPS_RESET:
		return OCRDMA_QPS_RST;
	case IBV_QPS_INIT:
		return OCRDMA_QPS_INIT;
	case IBV_QPS_RTR:
		return OCRDMA_QPS_RTR;
	case IBV_QPS_RTS:
		return OCRDMA_QPS_RTS;
	case IBV_QPS_SQD:
		return OCRDMA_QPS_SQD;
	case IBV_QPS_SQE:
		return OCRDMA_QPS_SQE;
	case IBV_QPS_ERR:
		return OCRDMA_QPS_ERR;
	case IBV_QPS_UNKNOWN:
		break;
	default:
		break;
	};
	return OCRDMA_QPS_ERR;
}

static int ocrdma_is_qp_in_sq_flushlist(struct ocrdma_cq *cq,
					struct ocrdma_qp *qp)
{
	struct ocrdma_qp *list_qp;
	struct ocrdma_qp *list_qp_tmp;
	int found = 0;
	list_for_each_safe(&cq->sq_head, list_qp, list_qp_tmp, sq_entry) {
		if (qp == list_qp) {
			found = 1;
			break;
		}
	}
	return found;
}

static int ocrdma_is_qp_in_rq_flushlist(struct ocrdma_cq *cq,
					struct ocrdma_qp *qp)
{
	struct ocrdma_qp *list_qp;
	struct ocrdma_qp *list_qp_tmp;
	int found = 0;
	list_for_each_safe(&cq->rq_head, list_qp, list_qp_tmp, rq_entry) {
		if (qp == list_qp) {
			found = 1;
			break;
		}
	}
	return found;
}

static void ocrdma_init_hwq_ptr(struct ocrdma_qp *qp)
{
	qp->sq.head = qp->sq.tail = 0;
	qp->rq.head = qp->rq.tail = 0;
	qp->dpp_q.head = qp->dpp_q.tail = 0;
	qp->dpp_q.free_cnt = qp->dpp_q.max_cnt;
}

static void ocrdma_del_flush_qp(struct ocrdma_qp *qp)
{
	int found = 0;
	struct ocrdma_device *dev = qp->dev;
	/* sync with any active CQ poll */

	pthread_spin_lock(&dev->flush_q_lock);
	found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp);
	if (found)
		list_del(&qp->sq_entry);
	if (!qp->srq) {
		found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp);
		if (found)
			list_del(&qp->rq_entry);
	}
	pthread_spin_unlock(&dev->flush_q_lock);
}

static void ocrdma_flush_qp(struct ocrdma_qp *qp)
{
	int found;

	pthread_spin_lock(&qp->dev->flush_q_lock);
	found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp);
	if (!found)
		list_add_tail(&qp->sq_cq->sq_head, &qp->sq_entry);
	if (!qp->srq) {
		found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp);
		if (!found)
			list_add_tail(&qp->rq_cq->rq_head, &qp->rq_entry);
	}
	pthread_spin_unlock(&qp->dev->flush_q_lock);
}

static int ocrdma_qp_state_machine(struct ocrdma_qp *qp,
				   enum ibv_qp_state new_ib_state)
{
	int status = 0;
	enum ocrdma_qp_state new_state;
	new_state = get_ocrdma_qp_state(new_ib_state);

	pthread_spin_lock(&qp->q_lock);

	if (new_state == qp->state) {
		pthread_spin_unlock(&qp->q_lock);
		return 1;
	}

	switch (qp->state) {
	case OCRDMA_QPS_RST:
		switch (new_state) {
		case OCRDMA_QPS_RST:
			break;
		case OCRDMA_QPS_INIT:
			/* init pointers to place wqe/rqe at start of hw q */
			ocrdma_init_hwq_ptr(qp);
			/* detach qp from the CQ flush list */
			ocrdma_del_flush_qp(qp);
			break;
		default:
			status = EINVAL;
			break;
		};
		break;
	case OCRDMA_QPS_INIT:
		/* qps: INIT->XXX */
		switch (new_state) {
		case OCRDMA_QPS_INIT:
			break;
		case OCRDMA_QPS_RTR:
			break;
		case OCRDMA_QPS_ERR:
			ocrdma_flush_qp(qp);
			break;
		default:
			/* invalid state change. */
			status = EINVAL;
			break;
		};
		break;
	case OCRDMA_QPS_RTR:
		/* qps: RTS->XXX */
		switch (new_state) {
		case OCRDMA_QPS_RTS:
			break;
		case OCRDMA_QPS_ERR:
			ocrdma_flush_qp(qp);
			break;
		default:
			/* invalid state change. */
			status = EINVAL;
			break;
		};
		break;
	case OCRDMA_QPS_RTS:
		/* qps: RTS->XXX */
		switch (new_state) {
		case OCRDMA_QPS_SQD:
		case OCRDMA_QPS_SQE:
			break;
		case OCRDMA_QPS_ERR:
			ocrdma_flush_qp(qp);
			break;
		default:
			/* invalid state change. */
			status = EINVAL;
			break;
		};
		break;
	case OCRDMA_QPS_SQD:
		/* qps: SQD->XXX */
		switch (new_state) {
		case OCRDMA_QPS_RTS:
		case OCRDMA_QPS_SQE:
		case OCRDMA_QPS_ERR:
			break;
		default:
			/* invalid state change. */
			status = EINVAL;
			break;
		};
		break;
	case OCRDMA_QPS_SQE:
		switch (new_state) {
		case OCRDMA_QPS_RTS:
		case OCRDMA_QPS_ERR:
			break;
		default:
			/* invalid state change. */
			status = EINVAL;
			break;
		};
		break;
	case OCRDMA_QPS_ERR:
		/* qps: ERR->XXX */
		switch (new_state) {
		case OCRDMA_QPS_RST:
			break;
		default:
			status = EINVAL;
			break;
		};
		break;
	default:
		status = EINVAL;
		break;
	};
	if (!status)
		qp->state = new_state;

	pthread_spin_unlock(&qp->q_lock);
	return status;
}

/*
 * ocrdma_modify_qp
 */
int ocrdma_modify_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr,
		     int attr_mask)
{
	struct ibv_modify_qp cmd = {};
	struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
	int status;

	status = ibv_cmd_modify_qp(ibqp, attr, attr_mask, &cmd, sizeof cmd);
	if ((!status) && (attr_mask & IBV_QP_STATE))
		ocrdma_qp_state_machine(qp, attr->qp_state);
	return status;
}

/*
 * ocrdma_query_qp
 */
int ocrdma_query_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr,
		    int attr_mask, struct ibv_qp_init_attr *init_attr)
{
	struct ibv_query_qp cmd;
	struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
	int status;

	status = ibv_cmd_query_qp(ibqp, attr, attr_mask,
				  init_attr, &cmd, sizeof(cmd));

	if (!status)
		ocrdma_qp_state_machine(qp, attr->qp_state);

	return status;
}

static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, int idx)
{
	int i = idx / 32;
	unsigned int mask = (1 << (idx % 32));

	if (srq->idx_bit_fields[i] & mask) {
		srq->idx_bit_fields[i] &= ~mask;
	} else {
		srq->idx_bit_fields[i] |= mask;
	}
}

static int ocrdma_srq_get_idx(struct ocrdma_srq *srq)
{
	int row = 0;
	int indx = 0;

	for (row = 0; row < srq->bit_fields_len; row++) {
		if (srq->idx_bit_fields[row]) {
			indx = ffs(srq->idx_bit_fields[row]);
			indx = (row * 32) + (indx - 1);
			if (indx >= srq->rq.max_cnt)
				assert(0);
			ocrdma_srq_toggle_bit(srq, indx);
			break;
		}
	}
	if (row == srq->bit_fields_len)
		assert(0);
	return indx + 1; /* Use the index from 1 */
}

static int ocrdma_dppq_credits(struct ocrdma_qp_hwq_info *q)
{
	return ((q->max_wqe_idx - q->head) + q->tail) % q->free_cnt;
}

static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q)
{
	return ((q->max_wqe_idx - q->head) + q->tail) % q->max_cnt;
}

static int is_hw_sq_empty(struct ocrdma_qp *qp)
{
	return ((qp->sq.tail == qp->sq.head) ? 1 : 0);
}

static inline int is_hw_rq_empty(struct ocrdma_qp *qp)
{
	return ((qp->rq.head == qp->rq.tail) ? 1 : 0);
}

static inline void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q)
{
	return q->va + (q->head * q->entry_size);
}

/*static inline void *ocrdma_wq_tail(struct ocrdma_qp_hwq_info *q)
{
	return q->va + (q->tail * q->entry_size);
}
*/

static inline void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info *q,
					     uint32_t idx)
{
	return q->va + (idx * q->entry_size);
}

static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info *q)
{
	q->head = (q->head + 1) & q->max_wqe_idx;
}

static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info *q)
{
	q->tail = (q->tail + 1) & q->max_wqe_idx;
}

static inline void ocrdma_hwq_inc_tail_by_idx(struct ocrdma_qp_hwq_info *q,
					      int idx)
{
	q->tail = (idx + 1) & q->max_wqe_idx;
}

static int is_cqe_valid(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe)
{
	int cqe_valid;
	cqe_valid = le32toh(cqe->flags_status_srcqpn) & OCRDMA_CQE_VALID;
	return (cqe_valid == cq->phase);
}

static int is_cqe_for_sq(struct ocrdma_cqe *cqe)
{
	return (le32toh(cqe->flags_status_srcqpn) &
		OCRDMA_CQE_QTYPE) ? 0 : 1;
}

static int is_cqe_imm(struct ocrdma_cqe *cqe)
{
	return (le32toh(cqe->flags_status_srcqpn) &
		OCRDMA_CQE_IMM) ? 1 : 0;
}

static int is_cqe_wr_imm(struct ocrdma_cqe *cqe)
{
	return (le32toh(cqe->flags_status_srcqpn) &
		OCRDMA_CQE_WRITE_IMM) ? 1 : 0;
}

static inline void ocrdma_srq_inc_tail(struct ocrdma_qp *qp,
				       struct ocrdma_cqe *cqe)
{
	int wqe_idx;

	wqe_idx = (le32toh(cqe->rq.buftag_qpn) >>
	    OCRDMA_CQE_BUFTAG_SHIFT) & qp->srq->rq.max_wqe_idx;

	if (wqe_idx < 1)
		assert(0);

	pthread_spin_lock(&qp->srq->q_lock);
	ocrdma_hwq_inc_tail(&qp->srq->rq);
	ocrdma_srq_toggle_bit(qp->srq, wqe_idx - 1);
	pthread_spin_unlock(&qp->srq->q_lock);
}

static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq)
{
	uint32_t cur_getp, stop_getp;
	struct ocrdma_cqe *cqe;
	uint32_t qpn = 0;
	int wqe_idx;

	pthread_spin_lock(&cq->cq_lock);

	/* traverse through the CQEs in the hw CQ,
	 * find the matching CQE for a given qp,
	 * mark the matching one discarded=1.
	 * discard the cqe.
	 * ring the doorbell in the poll_cq() as
	 * we don't complete out of order cqe.
	 */
	cur_getp = cq->getp;
	/* find up to when do we reap the cq.*/
	stop_getp = cur_getp;
	do {
		if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp)))
			break;

		cqe = cq->va + cur_getp;
		/* if (a) no valid cqe, or (b) done reading full hw cq, or
		 *    (c) qp_xq becomes empty.
		 * then exit
		 */
		qpn = le32toh(cqe->cmn.qpn) & OCRDMA_CQE_QPN_MASK;
		/* if previously discarded cqe found, skip that too.
		 * check for matching qp
		 */
		if ((qpn == 0) || (qpn != qp->id))
			goto skip_cqe;

		/* mark cqe discarded so that it is not picked up later
		 * in the poll_cq().
		 */
		if (is_cqe_for_sq(cqe)) {
			wqe_idx = (le32toh(cqe->wq.wqeidx) &
			    OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx;
			ocrdma_hwq_inc_tail_by_idx(&qp->sq, wqe_idx);
		} else {
			if (qp->srq)
				ocrdma_srq_inc_tail(qp, cqe);
			else
				ocrdma_hwq_inc_tail(&qp->rq);
		}

		/* discard by marking qp_id = 0 */
		cqe->cmn.qpn = 0;
skip_cqe:
		cur_getp = (cur_getp + 1) % cq->max_hw_cqe;

	} while (cur_getp != stop_getp);
	pthread_spin_unlock(&cq->cq_lock);
}

/*
 * ocrdma_destroy_qp
 */
int ocrdma_destroy_qp(struct ibv_qp *ibqp)
{
	int status = 0;
	struct ocrdma_qp *qp;
	struct ocrdma_device *dev;

	qp = get_ocrdma_qp(ibqp);
	dev = qp->dev;
	/*
	 * acquire CQ lock while destroy is in progress, in order to
	 * protect against proessing in-flight CQEs for this QP.
	 */
	pthread_spin_lock(&qp->sq_cq->cq_lock);

	if (qp->rq_cq && (qp->rq_cq != qp->sq_cq))
		pthread_spin_lock(&qp->rq_cq->cq_lock);

	_ocrdma_del_qpn_map(qp->dev, qp);

	if (qp->rq_cq && (qp->rq_cq != qp->sq_cq))
		pthread_spin_unlock(&qp->rq_cq->cq_lock);

	pthread_spin_unlock(&qp->sq_cq->cq_lock);

	if (qp->db_va)
		munmap((void *)qp->db_va, qp->db_size);
	if (qp->rq.va)
		munmap(qp->rq.va, qp->rq.len);
	if (qp->sq.va)
		munmap(qp->sq.va, qp->sq.len);

	/* ensure that CQEs for newly created QP (whose id may be same with
	 * one which just getting destroyed are same), don't get
	 * discarded until the old CQEs are discarded.
	 */
	pthread_mutex_lock(&dev->dev_lock);
	status = ibv_cmd_destroy_qp(ibqp);

	ocrdma_discard_cqes(qp, qp->sq_cq);
	ocrdma_discard_cqes(qp, qp->rq_cq);
	pthread_mutex_unlock(&dev->dev_lock);

	ocrdma_del_flush_qp(qp);

	pthread_spin_destroy(&qp->q_lock);
	if (qp->rqe_wr_id_tbl)
		free(qp->rqe_wr_id_tbl);
	if (qp->wqe_wr_id_tbl)
		free(qp->wqe_wr_id_tbl);
	if (qp->dpp_cq)
		ocrdma_destroy_cq(&qp->dpp_cq->ibv_cq);
	free(qp);

	return status;
}

static void ocrdma_ring_sq_db(struct ocrdma_qp *qp)
{
	__le32 db_val = htole32((qp->sq.dbid | (1 << 16)));

	udma_to_device_barrier();
	*(__le32 *) (((uint8_t *) qp->db_sq_va)) = db_val;
}

static void ocrdma_ring_rq_db(struct ocrdma_qp *qp)
{
	__le32 db_val = htole32((qp->rq.dbid | (1 << qp->db_shift)));

	udma_to_device_barrier();
	*(__le32 *) ((uint8_t *) qp->db_rq_va) = db_val;
}

static void ocrdma_ring_srq_db(struct ocrdma_srq *srq)
{
	__le32 db_val = htole32(srq->rq.dbid | (1 << srq->db_shift));

	udma_to_device_barrier();
	*(__le32 *) (srq->db_va) = db_val;
}

static void ocrdma_ring_cq_db(struct ocrdma_cq *cq, uint32_t armed,
			      int solicited, uint32_t num_cqe)
{
	uint32_t val;

	val = cq->cq_dbid & OCRDMA_DB_CQ_RING_ID_MASK;
	val |= ((cq->cq_dbid & OCRDMA_DB_CQ_RING_ID_EXT_MASK) <<
		OCRDMA_DB_CQ_RING_ID_EXT_MASK_SHIFT);

	if (armed)
		val |= (1 << OCRDMA_DB_CQ_REARM_SHIFT);
	if (solicited)
		val |= (1 << OCRDMA_DB_CQ_SOLICIT_SHIFT);
	val |= (num_cqe << OCRDMA_DB_CQ_NUM_POPPED_SHIFT);

	udma_to_device_barrier();
	*(__le32 *) ((uint8_t *) (cq->db_va) + OCRDMA_DB_CQ_OFFSET) =
	    htole32(val);
}

static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp,
				struct ocrdma_hdr_wqe *hdr,
				struct ibv_send_wr *wr)
{
	struct ocrdma_ewqe_ud_hdr *ud_hdr =
	    (struct ocrdma_ewqe_ud_hdr *)(hdr + 1);
	struct ocrdma_ah *ah = get_ocrdma_ah(wr->wr.ud.ah);

	ud_hdr->rsvd_dest_qpn = wr->wr.ud.remote_qpn;
	ud_hdr->qkey = wr->wr.ud.remote_qkey;
	ud_hdr->rsvd_ahid = ah->id;
	if (ah->isvlan)
		hdr->cw |= (OCRDMA_FLAG_AH_VLAN_PR <<
			    OCRDMA_WQE_FLAGS_SHIFT);
	ud_hdr->hdr_type = ah->hdr_type;
}

static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr,
			      struct ocrdma_sge *sge, int num_sge,
			      struct ibv_sge *sg_list)
{
	int i;
	for (i = 0; i < num_sge; i++) {
		sge[i].lrkey = sg_list[i].lkey;
		sge[i].addr_lo = sg_list[i].addr;
		sge[i].addr_hi = sg_list[i].addr >> 32;
		sge[i].len = sg_list[i].length;
		hdr->total_len += sg_list[i].length;
	}
	if (num_sge == 0)
		memset(sge, 0, sizeof(*sge));
}


static inline uint32_t ocrdma_sglist_len(struct ibv_sge *sg_list, int num_sge)
{
	uint32_t total_len = 0, i;

	for (i = 0; i < num_sge; i++)
		total_len += sg_list[i].length;
	return total_len;
}

static inline int ocrdma_build_inline_sges(struct ocrdma_qp *qp,
					   struct ocrdma_hdr_wqe *hdr,
					   struct ocrdma_sge *sge,
					   struct ibv_send_wr *wr,
					   uint32_t wqe_size)
{
	int i;
	char *dpp_addr;

	if (wr->send_flags & IBV_SEND_INLINE && qp->qp_type != IBV_QPT_UD) {
		hdr->total_len = ocrdma_sglist_len(wr->sg_list, wr->num_sge);
		if (hdr->total_len > qp->max_inline_data) {
			ocrdma_err
			("%s() supported_len=0x%x, unsupported len req=0x%x\n",
			__func__, qp->max_inline_data, hdr->total_len);
			return EINVAL;
		}

		dpp_addr = (char *)sge;
		for (i = 0; i < wr->num_sge; i++) {
			memcpy(dpp_addr,
				(void *)(unsigned long)wr->sg_list[i].addr,
				wr->sg_list[i].length);
			dpp_addr += wr->sg_list[i].length;
		}

		wqe_size += ROUND_UP_X(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES);
		if (0 == hdr->total_len)
			wqe_size += sizeof(struct ocrdma_sge);
		hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT);
	} else {
		ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);
		if (wr->num_sge)
			wqe_size += (wr->num_sge * sizeof(struct ocrdma_sge));
		else
			wqe_size += sizeof(struct ocrdma_sge);
		hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
	}
	hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
	return 0;
}

static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
			     struct ibv_send_wr *wr)
{
	int status;
	struct ocrdma_sge *sge;
	uint32_t wqe_size = sizeof(*hdr);

	if (qp->qp_type == IBV_QPT_UD) {
		wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr);
		ocrdma_build_ud_hdr(qp, hdr, wr);
		sge = (struct ocrdma_sge *)(hdr + 2);
	} else
		sge = (struct ocrdma_sge *)(hdr + 1);

	status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);

	return status;
}

static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
			      struct ibv_send_wr *wr)
{
	int status;
	struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);
	struct ocrdma_sge *sge = ext_rw + 1;
	uint32_t wqe_size = sizeof(*hdr) + sizeof(*ext_rw);

	status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
	if (status)
		return status;

	ext_rw->addr_lo = wr->wr.rdma.remote_addr;
	ext_rw->addr_hi = (wr->wr.rdma.remote_addr >> 32);
	ext_rw->lrkey = wr->wr.rdma.rkey;
	ext_rw->len = hdr->total_len;

	return 0;
}

static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
			      struct ibv_send_wr *wr)
{
	struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);
	struct ocrdma_sge *sge = ext_rw + 1;
	uint32_t wqe_size = ((wr->num_sge + 1) * sizeof(*sge)) + sizeof(*hdr);

	hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
	hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
	hdr->cw |= (OCRDMA_READ << OCRDMA_WQE_OPCODE_SHIFT);

	ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);

	ext_rw->addr_lo = wr->wr.rdma.remote_addr;
	ext_rw->addr_hi = (wr->wr.rdma.remote_addr >> 32);
	ext_rw->lrkey = wr->wr.rdma.rkey;
	ext_rw->len = hdr->total_len;

}

/* Dpp cq is single entry cq, we just need to read
 * wqe index from first 16 bits at 0th cqe index.
 */
static void ocrdma_poll_dpp_cq(struct ocrdma_qp *qp)
{
	struct ocrdma_cq *cq = qp->dpp_cq;
	struct ocrdma_dpp_cqe *cqe;
	int idx = 0;
	cqe = ((struct ocrdma_dpp_cqe *)cq->va);
	idx = cqe->wqe_idx_valid & OCRDMA_DPP_WQE_INDEX_MASK;

	if (idx != qp->dpp_prev_indx) {
		ocrdma_hwq_inc_tail_by_idx(&qp->dpp_q, idx);
		qp->dpp_prev_indx = idx;
	}
}

static uint32_t ocrdma_get_hdr_len(struct ocrdma_qp *qp,
				   struct ocrdma_hdr_wqe *hdr)
{
	uint32_t hdr_sz = sizeof(*hdr);
	if (qp->qp_type == IBV_QPT_UD)
		hdr_sz += sizeof(struct ocrdma_ewqe_ud_hdr);
	if (hdr->cw & (OCRDMA_WRITE << OCRDMA_WQE_OPCODE_SHIFT))
		hdr_sz += sizeof(struct ocrdma_sge);
	return hdr_sz / sizeof(uint32_t);
}

static void ocrdma_build_dpp_wqe(void *va, struct ocrdma_hdr_wqe *wqe,
				 uint32_t hdr_len)
{
	uint32_t pyld_len = (wqe->cw >> OCRDMA_WQE_SIZE_SHIFT) * 2;
	uint32_t i = 0;

	mmio_wc_start();

	/* convert WQE header to LE format */
	for (; i < hdr_len; i++)
		*((__le32 *) va + i) =
			htole32(*((uint32_t *) wqe + i));
	/* Convertion of data is done in HW */
	for (; i < pyld_len; i++)
		*((uint32_t *) va + i) = (*((uint32_t *) wqe + i));

	mmio_flush_writes();
}

static void ocrdma_post_dpp_wqe(struct ocrdma_qp *qp,
				struct ocrdma_hdr_wqe *hdr)
{
	if (qp->dpp_cq && ocrdma_dppq_credits(&qp->dpp_q) == 0)
		ocrdma_poll_dpp_cq(qp);
	if (!qp->dpp_cq || ocrdma_dppq_credits(&qp->dpp_q)) {
		ocrdma_build_dpp_wqe(qp->dpp_q.va, hdr,
				     ocrdma_get_hdr_len(qp, hdr));
		qp->wqe_wr_id_tbl[qp->sq.head].dpp_wqe = 1;
		qp->wqe_wr_id_tbl[qp->sq.head].dpp_wqe_idx = qp->dpp_q.head;
		/* if dpp cq is not enabled, we can post
		 * wqe as soon as we receive and adapter
		 * takes care of flow control.
		 */
		if (qp->dpp_cq)
			ocrdma_hwq_inc_head(&qp->dpp_q);
	} else
		qp->wqe_wr_id_tbl[qp->sq.head].dpp_wqe = 0;
}

/*
 * ocrdma_post_send
 */
int ocrdma_post_send(struct ibv_qp *ib_qp, struct ibv_send_wr *wr,
		     struct ibv_send_wr **bad_wr)
{
	int status = 0;
	struct ocrdma_qp *qp;
	struct ocrdma_hdr_wqe *hdr;

	qp = get_ocrdma_qp(ib_qp);

	pthread_spin_lock(&qp->q_lock);
	if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) {
		pthread_spin_unlock(&qp->q_lock);
		*bad_wr = wr;
		return EINVAL;
	}

	while (wr) {

		if (qp->qp_type == IBV_QPT_UD && (wr->opcode != IBV_WR_SEND &&
		    wr->opcode != IBV_WR_SEND_WITH_IMM)) {
			*bad_wr = wr;
			status = EINVAL;
			break;
		}

		if (ocrdma_hwq_free_cnt(&qp->sq) == 0 ||
		    wr->num_sge > qp->sq.max_sges) {
			*bad_wr = wr;
			status = ENOMEM;
			break;
		}
		hdr = ocrdma_hwq_head(&qp->sq);
		hdr->cw = 0;
		hdr->total_len = 0;
		if (wr->send_flags & IBV_SEND_SIGNALED || qp->signaled)
			hdr->cw = (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
		if (wr->send_flags & IBV_SEND_FENCE)
			hdr->cw |=
			    (OCRDMA_FLAG_FENCE_L << OCRDMA_WQE_FLAGS_SHIFT);
		if (wr->send_flags & IBV_SEND_SOLICITED)
			hdr->cw |=
			    (OCRDMA_FLAG_SOLICIT << OCRDMA_WQE_FLAGS_SHIFT);

		qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id;
		switch (wr->opcode) {
		case IBV_WR_SEND_WITH_IMM:
			hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);
			hdr->immdt = be32toh(wr->imm_data);
			SWITCH_FALLTHROUGH;
		case IBV_WR_SEND:
			hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);
			status = ocrdma_build_send(qp, hdr, wr);
			break;
		case IBV_WR_RDMA_WRITE_WITH_IMM:
			hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);
			hdr->immdt = be32toh(wr->imm_data);
			SWITCH_FALLTHROUGH;
		case IBV_WR_RDMA_WRITE:
			hdr->cw |= (OCRDMA_WRITE << OCRDMA_WQE_OPCODE_SHIFT);
			status = ocrdma_build_write(qp, hdr, wr);
			break;
		case IBV_WR_RDMA_READ:
			ocrdma_build_read(qp, hdr, wr);
			break;
		default:
			status = EINVAL;
			break;
		}
		if (status) {
			*bad_wr = wr;
			break;
		}
		if (wr->send_flags & IBV_SEND_SIGNALED || qp->signaled)
			qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1;
		else
			qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0;

		if (qp->dpp_enabled && (wr->send_flags & IBV_SEND_INLINE))
			ocrdma_post_dpp_wqe(qp, hdr);

		ocrdma_swap_cpu_to_le(hdr, ((hdr->cw >> OCRDMA_WQE_SIZE_SHIFT) &
				      OCRDMA_WQE_SIZE_MASK) *
				      OCRDMA_WQE_STRIDE);

		ocrdma_ring_sq_db(qp);

		/* update pointer, counter for next wr */
		ocrdma_hwq_inc_head(&qp->sq);
		wr = wr->next;
	}
	pthread_spin_unlock(&qp->q_lock);

	return status;
}

static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe, struct ibv_recv_wr *wr,
			     uint16_t tag)
{
	struct ocrdma_sge *sge;
	uint32_t wqe_size;

	if (wr->num_sge)
		wqe_size = (wr->num_sge * sizeof(*sge)) + sizeof(*rqe);
	else
		wqe_size = sizeof(*sge) + sizeof(*rqe);

	rqe->cw = ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
	rqe->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
	rqe->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
	rqe->total_len = 0;
	rqe->rsvd_tag = tag;
	sge = (struct ocrdma_sge *)(rqe + 1);
	ocrdma_build_sges(rqe, sge, wr->num_sge, wr->sg_list);
	ocrdma_swap_cpu_to_le(rqe, wqe_size);
}

/*
 * ocrdma_post_recv
 */
int ocrdma_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
		     struct ibv_recv_wr **bad_wr)
{
	int status = 0;
	struct ocrdma_qp *qp;
	struct ocrdma_hdr_wqe *rqe;

	qp = get_ocrdma_qp(ibqp);

	pthread_spin_lock(&qp->q_lock);
	if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) {
		pthread_spin_unlock(&qp->q_lock);
		*bad_wr = wr;
		return EINVAL;
	}

	while (wr) {
		if (ocrdma_hwq_free_cnt(&qp->rq) == 0 ||
		    wr->num_sge > qp->rq.max_sges) {
			status = ENOMEM;
			*bad_wr = wr;
			break;
		}
		rqe = ocrdma_hwq_head(&qp->rq);
		ocrdma_build_rqe(rqe, wr, 0);
		qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id;
		ocrdma_ring_rq_db(qp);

		/* update pointer, counter for next wr */
		ocrdma_hwq_inc_head(&qp->rq);
		wr = wr->next;
	}
	pthread_spin_unlock(&qp->q_lock);

	return status;
}

static enum ibv_wc_status ocrdma_to_ibwc_err(uint16_t status)
{
	enum ibv_wc_status ibwc_status = IBV_WC_GENERAL_ERR;
	switch (status) {
	case OCRDMA_CQE_GENERAL_ERR:
		ibwc_status = IBV_WC_GENERAL_ERR;
		break;
	case OCRDMA_CQE_LOC_LEN_ERR:
		ibwc_status = IBV_WC_LOC_LEN_ERR;
		break;
	case OCRDMA_CQE_LOC_QP_OP_ERR:
		ibwc_status = IBV_WC_LOC_QP_OP_ERR;
		break;
	case OCRDMA_CQE_LOC_EEC_OP_ERR:
		ibwc_status = IBV_WC_LOC_EEC_OP_ERR;
		break;
	case OCRDMA_CQE_LOC_PROT_ERR:
		ibwc_status = IBV_WC_LOC_PROT_ERR;
		break;
	case OCRDMA_CQE_WR_FLUSH_ERR:
		ibwc_status = IBV_WC_WR_FLUSH_ERR;
		break;
	case OCRDMA_CQE_BAD_RESP_ERR:
		ibwc_status = IBV_WC_BAD_RESP_ERR;
		break;
	case OCRDMA_CQE_LOC_ACCESS_ERR:
		ibwc_status = IBV_WC_LOC_ACCESS_ERR;
		break;
	case OCRDMA_CQE_REM_INV_REQ_ERR:
		ibwc_status = IBV_WC_REM_INV_REQ_ERR;
		break;
	case OCRDMA_CQE_REM_ACCESS_ERR:
		ibwc_status = IBV_WC_REM_ACCESS_ERR;
		break;
	case OCRDMA_CQE_REM_OP_ERR:
		ibwc_status = IBV_WC_REM_OP_ERR;
		break;
	case OCRDMA_CQE_RETRY_EXC_ERR:
		ibwc_status = IBV_WC_RETRY_EXC_ERR;
		break;
	case OCRDMA_CQE_RNR_RETRY_EXC_ERR:
		ibwc_status = IBV_WC_RNR_RETRY_EXC_ERR;
		break;
	case OCRDMA_CQE_LOC_RDD_VIOL_ERR:
		ibwc_status = IBV_WC_LOC_RDD_VIOL_ERR;
		break;
	case OCRDMA_CQE_REM_INV_RD_REQ_ERR:
		ibwc_status = IBV_WC_REM_INV_RD_REQ_ERR;
		break;
	case OCRDMA_CQE_REM_ABORT_ERR:
		ibwc_status = IBV_WC_REM_ABORT_ERR;
		break;
	case OCRDMA_CQE_INV_EECN_ERR:
		ibwc_status = IBV_WC_INV_EECN_ERR;
		break;
	case OCRDMA_CQE_INV_EEC_STATE_ERR:
		ibwc_status = IBV_WC_INV_EEC_STATE_ERR;
		break;
	case OCRDMA_CQE_FATAL_ERR:
		ibwc_status = IBV_WC_FATAL_ERR;
		break;
	case OCRDMA_CQE_RESP_TIMEOUT_ERR:
		ibwc_status = IBV_WC_RESP_TIMEOUT_ERR;
		break;
	default:
		ibwc_status = IBV_WC_GENERAL_ERR;
		break;
	};
	return ibwc_status;
}

static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ibv_wc *ibwc,
			     uint32_t wqe_idx)
{
	struct ocrdma_hdr_wqe_le *hdr;
	struct ocrdma_sge *rw;
	int opcode;

	hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx);

	ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid;

	/* Undo the hdr->cw swap */
	opcode = le32toh(hdr->cw) & OCRDMA_WQE_OPCODE_MASK;
	switch (opcode) {
	case OCRDMA_WRITE:
		ibwc->opcode = IBV_WC_RDMA_WRITE;
		break;
	case OCRDMA_READ:
		rw = (struct ocrdma_sge *)(hdr + 1);
		ibwc->opcode = IBV_WC_RDMA_READ;
		ibwc->byte_len = rw->len;
		break;
	case OCRDMA_SEND:
		ibwc->opcode = IBV_WC_SEND;
		break;
	default:
		ibwc->status = IBV_WC_GENERAL_ERR;
		ocrdma_err("%s() invalid opcode received = 0x%x\n",
			   __func__, le32toh(hdr->cw) & OCRDMA_WQE_OPCODE_MASK);
		break;
	};
}

static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp,
					  struct ocrdma_cqe *cqe)
{
	if (is_cqe_for_sq(cqe)) {
		cqe->flags_status_srcqpn =
		    htole32(le32toh(cqe->flags_status_srcqpn)
				     & ~OCRDMA_CQE_STATUS_MASK);
		cqe->flags_status_srcqpn =
		    htole32(le32toh(cqe->flags_status_srcqpn)
				     | (OCRDMA_CQE_WR_FLUSH_ERR <<
					OCRDMA_CQE_STATUS_SHIFT));
	} else {
		if (qp->qp_type == IBV_QPT_UD) {
			cqe->flags_status_srcqpn =
			    htole32(le32toh
					     (cqe->flags_status_srcqpn) &
					     ~OCRDMA_CQE_UD_STATUS_MASK);
			cqe->flags_status_srcqpn =
			    htole32(le32toh
					     (cqe->flags_status_srcqpn) |
					     (OCRDMA_CQE_WR_FLUSH_ERR <<
					      OCRDMA_CQE_UD_STATUS_SHIFT));
		} else {
			cqe->flags_status_srcqpn =
			    htole32(le32toh
					     (cqe->flags_status_srcqpn) &
					     ~OCRDMA_CQE_STATUS_MASK);
			cqe->flags_status_srcqpn =
			    htole32(le32toh
					     (cqe->flags_status_srcqpn) |
					     (OCRDMA_CQE_WR_FLUSH_ERR <<
					      OCRDMA_CQE_STATUS_SHIFT));
		}
	}
}

static int ocrdma_update_err_cqe(struct ibv_wc *ibwc, struct ocrdma_cqe *cqe,
				 struct ocrdma_qp *qp, int status)
{
	int expand = 0;

	ibwc->byte_len = 0;
	ibwc->qp_num = qp->id;
	ibwc->status = ocrdma_to_ibwc_err(status);

	ocrdma_flush_qp(qp);
	ocrdma_qp_state_machine(qp, IBV_QPS_ERR);

	/* if wqe/rqe pending for which cqe needs to be returned,
	 * trigger inflating it.
	 */
	if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) {
		expand = 1;
		ocrdma_set_cqe_status_flushed(qp, cqe);
	}
	return expand;
}

static int ocrdma_update_err_rcqe(struct ibv_wc *ibwc, struct ocrdma_cqe *cqe,
				  struct ocrdma_qp *qp, int status)
{
	ibwc->opcode = IBV_WC_RECV;
	ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
	ocrdma_hwq_inc_tail(&qp->rq);

	return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
}

static int ocrdma_update_err_scqe(struct ibv_wc *ibwc, struct ocrdma_cqe *cqe,
				  struct ocrdma_qp *qp, int status)
{
	ocrdma_update_wc(qp, ibwc, qp->sq.tail);
	ocrdma_hwq_inc_tail(&qp->sq);

	return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
}

static int ocrdma_poll_err_scqe(struct ocrdma_qp *qp,
				struct ocrdma_cqe *cqe, struct ibv_wc *ibwc,
				int *polled, int *stop)
{
	int expand;
	int status = (le32toh(cqe->flags_status_srcqpn) &
		      OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;

	/* when hw sq is empty, but rq is not empty, so we continue
	 * to keep the cqe in order to get the cq event again.
	 */
	if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) {
		/* when cq for rq and sq is same, it is safe to return
		 * flush cqe for RQEs.
		 */
		if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
			*polled = 1;
			status = OCRDMA_CQE_WR_FLUSH_ERR;
			expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
		} else {
			*polled = 0;
			*stop = 1;
			expand = 0;
		}
	} else if (is_hw_sq_empty(qp)) {
		/* Do nothing */
		expand = 0;
		*polled = 0;
		*stop = 0;
	} else {
		*polled = 1;
		expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
	}
	return expand;
}

static int ocrdma_poll_success_scqe(struct ocrdma_qp *qp,
				    struct ocrdma_cqe *cqe,
				    struct ibv_wc *ibwc, int *polled)
{
	int expand = 0;
	int tail = qp->sq.tail;
	uint32_t wqe_idx;

	if (!qp->wqe_wr_id_tbl[tail].signaled) {
		*polled = 0;	/* WC cannot be consumed yet */
	} else {
		ibwc->status = IBV_WC_SUCCESS;
		ibwc->wc_flags = 0;
		ibwc->qp_num = qp->id;
		ocrdma_update_wc(qp, ibwc, tail);
		*polled = 1;
	}

	wqe_idx = (le32toh(cqe->wq.wqeidx) &
	    OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx;
	if (tail != wqe_idx)	/* CQE cannot be consumed yet */
		expand = 1;	/* Coallesced CQE */

	ocrdma_hwq_inc_tail(&qp->sq);
	return expand;
}

static int ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
			    struct ibv_wc *ibwc, int *polled, int *stop)
{
	int status, expand;

	status = (le32toh(cqe->flags_status_srcqpn) &
		  OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;

	if (status == OCRDMA_CQE_SUCCESS)
		expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled);
	else
		expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop);
	return expand;
}

static int ocrdma_update_ud_rcqe(struct ibv_wc *ibwc, struct ocrdma_cqe *cqe)
{
	int status;

	status = (le32toh(cqe->flags_status_srcqpn) &
		  OCRDMA_CQE_UD_STATUS_MASK) >> OCRDMA_CQE_UD_STATUS_SHIFT;
	ibwc->src_qp = le32toh(cqe->flags_status_srcqpn) &
	    OCRDMA_CQE_SRCQP_MASK;
	ibwc->pkey_index = le32toh(cqe->ud.rxlen_pkey) &
	    OCRDMA_CQE_PKEY_MASK;
	ibwc->wc_flags = IBV_WC_GRH;
	ibwc->byte_len = (le32toh(cqe->ud.rxlen_pkey) >>
			  OCRDMA_CQE_UD_XFER_LEN_SHIFT);
	return status;
}

static void ocrdma_update_free_srq_cqe(struct ibv_wc *ibwc,
				       struct ocrdma_cqe *cqe,
				       struct ocrdma_qp *qp)
{
	struct ocrdma_srq *srq = NULL;
	uint32_t wqe_idx;

	srq = get_ocrdma_srq(qp->ibv_qp.srq);
#if !defined(SKH_A0_WORKAROUND) /* BUG 113416 */
	wqe_idx = (le32toh(cqe->rq.buftag_qpn) >>
	    OCRDMA_CQE_BUFTAG_SHIFT) & srq->rq.max_wqe_idx;
#else
	wqe_idx = (le32toh(cqe->flags_status_srcqpn)) & 0xFFFF;
#endif
	if (wqe_idx < 1)
		assert(0);
	ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx];

	pthread_spin_lock(&srq->q_lock);
	ocrdma_srq_toggle_bit(srq, wqe_idx - 1);
	pthread_spin_unlock(&srq->q_lock);

	ocrdma_hwq_inc_tail(&srq->rq);
}

static int ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
				struct ibv_wc *ibwc, int *polled, int *stop,
				int status)
{
	int expand;

	/* when hw_rq is empty, but wq is not empty, so continue
	 * to keep the cqe to get the cq event again.
	 */
	if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) {
		if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
			*polled = 1;
			status = OCRDMA_CQE_WR_FLUSH_ERR;
			expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
		} else {
			*polled = 0;
			*stop = 1;
			expand = 0;
		}
	} else if (is_hw_rq_empty(qp)) {
		/* Do nothing */
		expand = 0;
		*polled = 0;
		*stop = 0;
	} else {
		*polled = 1;
		expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
	}
	return expand;
}

static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp,
				     struct ocrdma_cqe *cqe,
				     struct ibv_wc *ibwc)
{
	ibwc->opcode = IBV_WC_RECV;
	ibwc->qp_num = qp->id;
	ibwc->status = IBV_WC_SUCCESS;

	if (qp->qp_type == IBV_QPT_UD)
		ocrdma_update_ud_rcqe(ibwc, cqe);
	else
		ibwc->byte_len = le32toh(cqe->rq.rxlen);

	if (is_cqe_imm(cqe)) {
		ibwc->imm_data = htobe32(le32toh(cqe->rq.lkey_immdt));
		ibwc->wc_flags |= IBV_WC_WITH_IMM;
	} else if (is_cqe_wr_imm(cqe)) {
		ibwc->opcode = IBV_WC_RECV_RDMA_WITH_IMM;
		ibwc->imm_data = htobe32(le32toh(cqe->rq.lkey_immdt));
		ibwc->wc_flags |= IBV_WC_WITH_IMM;
	}
	if (qp->ibv_qp.srq)
		ocrdma_update_free_srq_cqe(ibwc, cqe, qp);
	else {
		ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
		ocrdma_hwq_inc_tail(&qp->rq);
	}
}

static int ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
			    struct ibv_wc *ibwc, int *polled, int *stop)
{
	int status;
	int expand = 0;

	ibwc->wc_flags = 0;
	if (qp->qp_type == IBV_QPT_UD)
		status = (le32toh(cqe->flags_status_srcqpn) &
			  OCRDMA_CQE_UD_STATUS_MASK) >>
				OCRDMA_CQE_UD_STATUS_SHIFT;
	else
		status = (le32toh(cqe->flags_status_srcqpn) &
			  OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;

	if (status == OCRDMA_CQE_SUCCESS) {
		*polled = 1;
		ocrdma_poll_success_rcqe(qp, cqe, ibwc);
	} else {
		expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop,
					      status);
	}
	return expand;
}

static void ocrdma_change_cq_phase(struct ocrdma_cq *cq,
				   struct ocrdma_cqe *cqe, uint16_t cur_getp)
{
	if (cq->phase_change) {
		if (cur_getp == 0)
			cq->phase = (~cq->phase & OCRDMA_CQE_VALID);
	} else
		cqe->flags_status_srcqpn = 0;	/* clear valid bit */
}

static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries,
			    struct ibv_wc *ibwc)
{
	uint16_t qpn = 0;
	int i = 0;
	int expand = 0;
	int polled_hw_cqes = 0;
	struct ocrdma_qp *qp = NULL;
	struct ocrdma_device *dev = cq->dev;
	struct ocrdma_cqe *cqe;
	uint16_t cur_getp;
	int polled = 0;
	int stop = 0;

	cur_getp = cq->getp;
	while (num_entries) {
		cqe = cq->va + cur_getp;
		/* check whether valid cqe or not */
		if (!is_cqe_valid(cq, cqe))
			break;
		qpn = (le32toh(cqe->cmn.qpn) & OCRDMA_CQE_QPN_MASK);
		/* ignore discarded cqe */
		if (qpn == 0)
			goto skip_cqe;
		qp = dev->qp_tbl[qpn];
		if (qp == NULL) {
			ocrdma_err("%s() cqe for invalid qpn= 0x%x received.\n",
				   __func__, qpn);
			goto skip_cqe;
		}

		if (is_cqe_for_sq(cqe)) {
			expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled,
						  &stop);
		} else {
			expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled,
						  &stop);
		}
		if (expand)
			goto expand_cqe;
		if (stop)
			goto stop_cqe;
		/* clear qpn to avoid duplicate processing by discard_cqe() */
		cqe->cmn.qpn = 0;
skip_cqe:
		polled_hw_cqes += 1;
		cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
		ocrdma_change_cq_phase(cq, cqe, cur_getp);
expand_cqe:
		if (polled) {
			num_entries -= 1;
			i += 1;
			ibwc = ibwc + 1;
			polled = 0;
		}
	}
stop_cqe:
	cq->getp = cur_getp;
	if (cq->deferred_arm || polled_hw_cqes) {
		ocrdma_ring_cq_db(cq, cq->deferred_arm,
				  cq->deferred_sol, polled_hw_cqes);
		cq->deferred_arm = 0;
		cq->deferred_sol = 0;
	}

	return i;
}

static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries,
			      struct ocrdma_qp *qp, struct ibv_wc *ibwc)
{
	int err_cqes = 0;

	while (num_entries) {
		if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp))
			break;
		if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) {
			ocrdma_update_wc(qp, ibwc, qp->sq.tail);
			ocrdma_hwq_inc_tail(&qp->sq);
		} else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) {
			ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
			ocrdma_hwq_inc_tail(&qp->rq);
		} else
			return err_cqes;
		ibwc->byte_len = 0;
		ibwc->status = IBV_WC_WR_FLUSH_ERR;
		ibwc = ibwc + 1;
		err_cqes += 1;
		num_entries -= 1;
	}
	return err_cqes;
}

/*
 * ocrdma_poll_cq
 */
int ocrdma_poll_cq(struct ibv_cq *ibcq, int num_entries, struct ibv_wc *wc)
{
	struct ocrdma_cq *cq;
	int cqes_to_poll = num_entries;
	int num_os_cqe = 0, err_cqes = 0;
	struct ocrdma_qp *qp;
	struct ocrdma_qp *qp_tmp;

	cq = get_ocrdma_cq(ibcq);
	pthread_spin_lock(&cq->cq_lock);
	num_os_cqe = ocrdma_poll_hwcq(cq, num_entries, wc);
	pthread_spin_unlock(&cq->cq_lock);
	cqes_to_poll -= num_os_cqe;

	if (cqes_to_poll) {
		wc = wc + num_os_cqe;
		pthread_spin_lock(&cq->dev->flush_q_lock);
		list_for_each_safe(&cq->sq_head, qp, qp_tmp, sq_entry) {
			if (cqes_to_poll == 0)
				break;
			err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc);
			cqes_to_poll -= err_cqes;
			num_os_cqe += err_cqes;
			wc = wc + err_cqes;
		}
		pthread_spin_unlock(&cq->dev->flush_q_lock);
	}
	return num_os_cqe;
}

/*
 * ocrdma_arm_cq
 */
int ocrdma_arm_cq(struct ibv_cq *ibcq, int solicited)
{
	struct ocrdma_cq *cq;

	cq = get_ocrdma_cq(ibcq);
	pthread_spin_lock(&cq->cq_lock);

	if (cq->first_arm) {
		ocrdma_ring_cq_db(cq, 1, solicited, 0);
		cq->first_arm = 0;
	}

	cq->deferred_arm = 1;
	cq->deferred_sol = solicited;

	pthread_spin_unlock(&cq->cq_lock);

	return 0;
}

/*
 * ocrdma_post_srq_recv
 */
int ocrdma_post_srq_recv(struct ibv_srq *ibsrq, struct ibv_recv_wr *wr,
			 struct ibv_recv_wr **bad_wr)
{
	int status = 0;
	uint16_t tag;
	struct ocrdma_srq *srq;
	struct ocrdma_hdr_wqe *rqe;

	srq = get_ocrdma_srq(ibsrq);
	pthread_spin_lock(&srq->q_lock);
	while (wr) {
		if (ocrdma_hwq_free_cnt(&srq->rq) == 0 ||
		    wr->num_sge > srq->rq.max_sges) {
			status = ENOMEM;
			*bad_wr = wr;
			break;
		}
		rqe = ocrdma_hwq_head(&srq->rq);
		tag = ocrdma_srq_get_idx(srq);
		ocrdma_build_rqe(rqe, wr, tag);
		srq->rqe_wr_id_tbl[tag] = wr->wr_id;

		ocrdma_ring_srq_db(srq);

		/* update pointer, counter for next wr */
		ocrdma_hwq_inc_head(&srq->rq);
		wr = wr->next;
	}
	pthread_spin_unlock(&srq->q_lock);
	return status;
}

/*
 * ocrdma_create_ah
 */
struct ibv_ah *ocrdma_create_ah(struct ibv_pd *ibpd, struct ibv_ah_attr *attr)
{
	int status;
	int ahtbl_idx;
	struct ocrdma_pd *pd;
	struct ocrdma_ah *ah;
	struct ib_uverbs_create_ah_resp resp;

	pd = get_ocrdma_pd(ibpd);
	ah = malloc(sizeof *ah);
	if (!ah)
		return NULL;
	bzero(ah, sizeof *ah);
	ah->pd = pd;

	ahtbl_idx = ocrdma_alloc_ah_tbl_id(pd->uctx);
	if (ahtbl_idx < 0)
		goto tbl_err;
	attr->dlid = ahtbl_idx;
	memset(&resp, 0, sizeof(resp));
	status = ibv_cmd_create_ah(ibpd, &ah->ibv_ah, attr, &resp, sizeof(resp));
	if (status)
		goto cmd_err;

	ah->id = pd->uctx->ah_tbl[ahtbl_idx] & OCRDMA_AH_ID_MASK;
	ah->isvlan = (pd->uctx->ah_tbl[ahtbl_idx] >>
			OCRDMA_AH_VLAN_VALID_SHIFT);
	ah->hdr_type = ((pd->uctx->ah_tbl[ahtbl_idx] >> OCRDMA_AH_L3_TYPE_SHIFT)
			& OCRDMA_AH_L3_TYPE_MASK);

	return &ah->ibv_ah;
cmd_err:
	ocrdma_free_ah_tbl_id(pd->uctx, ahtbl_idx);
tbl_err:
	free(ah);
	return NULL;
}

/*
 * ocrdma_destroy_ah
 */
int ocrdma_destroy_ah(struct ibv_ah *ibah)
{
	int status;
	struct ocrdma_ah *ah;

	ah = get_ocrdma_ah(ibah);

	status = ibv_cmd_destroy_ah(ibah);
	ocrdma_free_ah_tbl_id(ah->pd->uctx, ah->id);
	free(ah);
	return status;
}

/*
 * ocrdma_attach_mcast
 */
int ocrdma_attach_mcast(struct ibv_qp *ibqp, const union ibv_gid *gid,
			uint16_t lid)
{
	return ibv_cmd_attach_mcast(ibqp, gid, lid);
}

/*
 * ocrdma_detach_mcast
 */
int ocrdma_detach_mcast(struct ibv_qp *ibqp, const union ibv_gid *gid,
			uint16_t lid)
{
	return ibv_cmd_detach_mcast(ibqp, gid, lid);
}