Merge pull request #1793 from yishaih/misc
[mlx4, mthca]: Fix signed-overflow as part of cq cleanup
diff --git a/debian/ibverbs-providers.symbols b/debian/ibverbs-providers.symbols
index 40e627d..044b4c9 100644
--- a/debian/ibverbs-providers.symbols
+++ b/debian/ibverbs-providers.symbols
@@ -211,8 +211,10 @@
libmana.so.1 ibverbs-providers #MINVER#
* Build-Depends-Package: libibverbs-dev
MANA_1.0@MANA_1.0 41
+ MANA_1.1@MANA_1.1 65
manadv_init_obj@MANA_1.0 41
manadv_set_context_attr@MANA_1.0 41
+ manadv_alloc_pd@MANA_1.1 65
libionic.so.1 ibverbs-providers #MINVER#
* Build-Depends-Package: libibverbs-dev
IONIC_1.0@IONIC_1.0 59
diff --git a/kernel-headers/rdma/mana-abi.h b/kernel-headers/rdma/mana-abi.h
index a75bf32..32cbbfc 100644
--- a/kernel-headers/rdma/mana-abi.h
+++ b/kernel-headers/rdma/mana-abi.h
@@ -25,7 +25,7 @@
struct mana_ib_create_cq {
__aligned_u64 buf_addr;
- __u16 flags;
+ __u16 comp_mask;
__u16 reserved0;
__u32 reserved1;
};
@@ -57,6 +57,17 @@
__u32 queue_id[4];
};
+struct mana_ib_create_uc_qp {
+ __aligned_u64 queue_buf[3];
+ __u32 queue_size[3];
+ __u32 comp_mask;
+};
+
+struct mana_ib_create_uc_qp_resp {
+ __u32 queue_id[3];
+ __u32 reserved;
+};
+
struct mana_ib_create_wq {
__aligned_u64 wq_buf_addr;
__u32 wq_buf_size;
@@ -87,4 +98,26 @@
struct rss_resp_entry entries[64];
};
+enum mana_ib_ucontext_support {
+ MANA_IB_UCNTX_ALLOC_PDN_SUPPORT = 1 << 0,
+};
+
+struct mana_ib_alloc_ucontext_resp {
+ __aligned_u64 comp_mask;
+};
+
+enum mana_ib_create_pd_flags {
+ MANA_IB_PD_SHORT_PDN = 1 << 0,
+};
+
+struct mana_ib_alloc_pd {
+ __u32 comp_mask;
+ __u32 reserved;
+};
+
+struct mana_ib_alloc_pd_resp {
+ __u32 pdn;
+ __u32 reserved;
+};
+
#endif
diff --git a/providers/mana/CMakeLists.txt b/providers/mana/CMakeLists.txt
index 05011be..3ae2b02 100644
--- a/providers/mana/CMakeLists.txt
+++ b/providers/mana/CMakeLists.txt
@@ -1,5 +1,5 @@
rdma_shared_provider(mana libmana.map
- 1 1.0.${PACKAGE_VERSION}
+ 1 1.1.${PACKAGE_VERSION}
mana.c
manadv.c
qp.c
diff --git a/providers/mana/cq.c b/providers/mana/cq.c
index 367d5a2..09da80c 100644
--- a/providers/mana/cq.c
+++ b/providers/mana/cq.c
@@ -165,7 +165,7 @@
cmd_drv = &cmd.drv_payload;
cmd_drv->buf_addr = (uintptr_t)cq->buf;
- cmd_drv->flags = flags;
+ cmd_drv->comp_mask = flags;
resp.cqid = UINT32_MAX;
ret = ibv_cmd_create_cq(context, cq->cqe, channel, comp_vector,
@@ -284,6 +284,25 @@
return produced;
}
+static inline int handle_requester_cqe(struct mana_qp *qp, struct gdma_cqe *cqe, struct ibv_wc *wc)
+{
+ struct mana_gdma_queue *send_queue = mana_ib_get_sreq(qp);
+ struct shadow_wqe_header *wqe;
+ int produced = 0;
+
+ while (!produced && (wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq)) != NULL) {
+ send_queue->cons_idx += wqe->posted_wqe_size_in_bu;
+ send_queue->cons_idx &= GDMA_QUEUE_OFFSET_MASK;
+ if (wqe->flags != MANA_NO_SIGNAL_WC) {
+ fill_verbs_from_shadow_wqe(qp, wc, wqe);
+ produced++;
+ }
+ shadow_queue_advance_consumer(&qp->shadow_sq);
+ }
+
+ return produced;
+}
+
static inline int handle_rc_requester_cqe(struct mana_qp *qp, struct gdma_cqe *cqe,
struct ibv_wc *wc, int nwc, bool *consumed)
{
@@ -443,6 +462,8 @@
return handle_error_cqe(qp, cqe, wc, nwc, consumed);
else if (cqe->rdma_cqe.cqe_type == CQE_TYPE_ARMED_CMPL)
return handle_rc_requester_cqe(qp, cqe, wc, nwc, consumed);
+ else if (cqe->is_sq && cqe->rdma_cqe.cqe_type == CQE_TYPE_UD_SEND)
+ return handle_requester_cqe(qp, cqe, wc);
else
return handle_responder_cqe(qp, cqe, wc);
}
diff --git a/providers/mana/doorbells.h b/providers/mana/doorbells.h
index d805cdb..7651e76 100644
--- a/providers/mana/doorbells.h
+++ b/providers/mana/doorbells.h
@@ -87,6 +87,16 @@
mmio_flush_writes();
}
+/* Has HW already produced the CQE at this index? Owner bits are written by HW,
+ * so a match proves idx < cur_cqe. Only meaningful for idx >= cq->head.
+ */
+static inline bool gdma_cq_idx_produced(struct mana_cq *cq, uint32_t idx)
+{
+ struct gdma_cqe *cqe = ((struct gdma_cqe *)cq->buf) + (idx % cq->cqe);
+
+ return cqe->owner_bits == ((idx / cq->cqe) & CQ_OWNER_MASK);
+}
+
static inline void gdma_ring_cq_doorbell(struct mana_cq *cq, uint8_t arm)
{
union gdma_doorbell_entry e;
@@ -94,8 +104,14 @@
uint32_t max_credit = cq->cqe << (GDMA_CQE_OWNER_BITS - 1);
if (cq->poll_credit >= max_credit) {
- // To address the use-case of ibv that re-arms the CQ without polling
- cq->poll_credit++;
+ // To address the use-case of ibv that re-arms the CQ without polling.
+ // (prod_idx + poll_credit - max_credit) is the index last given to HW,
+ // so only claim the next one once HW has produced it. Nothing produced
+ // implies no notification fired, so the CQ is still armed.
+ if (gdma_cq_idx_produced(cq, prod_idx + cq->poll_credit - max_credit))
+ cq->poll_credit++;
+ else
+ return;
} else {
// Set index of already polled CQE for unarm
cq->poll_credit = max_credit - (arm ? 0 : 1);
diff --git a/providers/mana/libmana.map b/providers/mana/libmana.map
index ab66295..24af016 100644
--- a/providers/mana/libmana.map
+++ b/providers/mana/libmana.map
@@ -6,3 +6,8 @@
manadv_init_obj;
local: *;
};
+
+MANA_1.1 {
+ global:
+ manadv_alloc_pd;
+} MANA_1.0;
diff --git a/providers/mana/man/CMakeLists.txt b/providers/mana/man/CMakeLists.txt
index 24f1859..3458443 100644
--- a/providers/mana/man/CMakeLists.txt
+++ b/providers/mana/man/CMakeLists.txt
@@ -2,4 +2,5 @@
manadv.7.md
manadv_init_obj.3.md
manadv_set_context_attr.3.md
+ manadv_alloc_pd.3.md
)
diff --git a/providers/mana/man/manadv_alloc_pd.3.md b/providers/mana/man/manadv_alloc_pd.3.md
new file mode 100644
index 0000000..b121ea2
--- /dev/null
+++ b/providers/mana/man/manadv_alloc_pd.3.md
@@ -0,0 +1,36 @@
+---
+layout: page
+title: manadv_alloc_pd
+section: 3
+tagline: Verbs
+---
+
+# NAME
+manadv_alloc_pd \- Create a MANA specific PD for the RDMA device context.
+
+# SYNOPSIS"
+```c
+#include <infiniband/manadv.h>
+
+struct ibv_pd *manadv_alloc_pd(struct ibv_context *context, uint32_t flags);
+```
+
+# DESCRIPTION
+**manadv_alloc_pd()** allocates a PD for the RDMA device context with additional
+creation flags.
+
+# ARGUMENTS
+*context*
+: RDMA device context to work on.
+
+*flags*
+: A bitwise OR of the various values described below.
+
+ MANADV_PD_FLAGS_SHORT_PDN:
+ allocates a PD with 16 bit PDN.
+
+# RETURN VALUE
+returns a pointer to the allocated PD, or NULL if the request fails.
+
+# AUTHORS
+Konstantin Taranov <kotaranov@microsoft.com>
diff --git a/providers/mana/man/manadv_init_obj.3.md b/providers/mana/man/manadv_init_obj.3.md
index 575ea34..a4dd712 100644
--- a/providers/mana/man/manadv_init_obj.3.md
+++ b/providers/mana/man/manadv_init_obj.3.md
@@ -48,6 +48,10 @@
void *db_page;
};
+struct manadv_pd {
+ uint32_t pdn;
+};
+
struct manadv_obj {
struct {
struct ibv_qp *in;
@@ -63,6 +67,11 @@
struct ibv_wq *in;
struct manadv_rwq *out;
} rwq;
+
+ struct {
+ struct ibv_pd *in;
+ struct manadv_pd *out;
+ } pd;
};
```
@@ -74,6 +83,7 @@
MANADV_OBJ_QP = 1 << 0,
MANADV_OBJ_CQ = 1 << 1,
MANADV_OBJ_RWQ = 1 << 2,
+ MANADV_OBJ_PD = 1 << 3,
};
```
# RETURN VALUE
diff --git a/providers/mana/mana.c b/providers/mana/mana.c
index a59248b..87411d4 100644
--- a/providers/mana/mana.c
+++ b/providers/mana/mana.c
@@ -21,9 +21,10 @@
#include "mana.h"
DECLARE_DRV_CMD(mana_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT, empty,
- empty);
+ mana_ib_alloc_ucontext_resp);
-DECLARE_DRV_CMD(mana_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, empty, empty);
+DECLARE_DRV_CMD(mana_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, mana_ib_alloc_pd,
+ mana_ib_alloc_pd_resp);
static const struct verbs_match_ent hca_table[] = {
VERBS_DRIVER_ID(RDMA_DRIVER_MANA),
@@ -114,19 +115,34 @@
return ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd));
}
-struct ibv_pd *mana_alloc_pd(struct ibv_context *context)
+struct ibv_pd *mana_alloc_pd_ex(struct ibv_context *context, uint32_t flags)
{
- struct ibv_alloc_pd cmd;
- struct mana_alloc_pd_resp resp;
+ struct mana_context *mctx = to_mctx(context);
+ struct mana_alloc_pd cmd = {};
+ struct mana_ib_alloc_pd *cmd_drv = &cmd.drv_payload;
+ struct mana_alloc_pd_resp resp = {};
+ size_t cmd_size = sizeof(cmd.ibv_cmd); /* v0 size */
struct mana_pd *pd;
int ret;
+ if ((flags & MANADV_PD_FLAGS_SHORT_PDN) &&
+ !(mctx->comp_mask & MANA_IB_UCNTX_ALLOC_PDN_SUPPORT)) {
+ errno = EOPNOTSUPP;
+ return NULL;
+ }
+
pd = calloc(1, sizeof(*pd));
if (!pd)
return NULL;
- ret = ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof(cmd),
- &resp.ibv_resp, sizeof(resp));
+ if (mctx->comp_mask & MANA_IB_UCNTX_ALLOC_PDN_SUPPORT)
+ cmd_size = sizeof(cmd); /* v1 size */
+
+ if (flags & MANADV_PD_FLAGS_SHORT_PDN)
+ cmd_drv->comp_mask |= MANA_IB_PD_SHORT_PDN;
+
+ ret = ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd.ibv_cmd, cmd_size,
+ &resp.ibv_resp, sizeof(resp));
if (ret) {
verbs_err(verbs_get_ctx(context), "Failed to allocate PD\n");
errno = ret;
@@ -134,9 +150,16 @@
return NULL;
}
+ pd->pdn = resp.pdn;
+
return &pd->ibv_pd;
}
+static struct ibv_pd *mana_alloc_pd(struct ibv_context *context)
+{
+ return mana_alloc_pd_ex(context, 0);
+}
+
struct ibv_pd *
mana_alloc_parent_domain(struct ibv_context *context,
struct ibv_parent_domain_init_attr *attr)
@@ -452,7 +475,7 @@
{
int ret, i;
struct mana_context *context;
- struct mana_alloc_ucontext_resp resp;
+ struct mana_alloc_ucontext_resp resp = {};
struct ibv_get_context cmd;
context = verbs_init_and_alloc_context(ibdev, cmd_fd, context, ibv_ctx,
@@ -468,6 +491,8 @@
goto free_ctx;
}
+ context->comp_mask = resp.drv_payload.comp_mask;
+
verbs_set_ops(&context->ibv_ctx, &mana_ctx_ops);
pthread_mutex_init(&context->qp_table_mutex, NULL);
diff --git a/providers/mana/mana.h b/providers/mana/mana.h
index 1f8a147..98318d1 100644
--- a/providers/mana/mana.h
+++ b/providers/mana/mana.h
@@ -49,7 +49,8 @@
USER_RNIC_SEND_QUEUE_RESPONDER = 1,
USER_RNIC_RECV_QUEUE_REQUESTER = 2,
USER_RNIC_RECV_QUEUE_RESPONDER = 3,
- USER_RNIC_QUEUE_TYPE_MAX = 4,
+ USER_RNIC_SEND_QUEUE_MM = 4,
+ USER_RNIC_QUEUE_TYPE_MAX = 5,
};
#define QUEUE_TYPE_MASK 0x3
@@ -89,6 +90,7 @@
struct mana_table qp_rtable[MANA_QP_TABLE_SIZE];
struct mana_table qp_stable[MANA_QP_TABLE_SIZE];
pthread_mutex_t qp_table_mutex;
+ uint64_t comp_mask;
struct manadv_ctx_allocators extern_alloc;
void *db_page;
@@ -196,6 +198,7 @@
struct mana_pd {
struct ibv_pd ibv_pd;
+ uint32_t pdn;
struct mana_pd *mprotection_domain;
};
@@ -216,7 +219,7 @@
int mana_query_port(struct ibv_context *context, uint8_t port,
struct ibv_port_attr *attr);
-struct ibv_pd *mana_alloc_pd(struct ibv_context *context);
+struct ibv_pd *mana_alloc_pd_ex(struct ibv_context *context, uint32_t flags);
struct ibv_pd *
mana_alloc_parent_domain(struct ibv_context *context,
struct ibv_parent_domain_init_attr *attr);
diff --git a/providers/mana/manadv.c b/providers/mana/manadv.c
index 4b40d05..2392adf 100644
--- a/providers/mana/manadv.c
+++ b/providers/mana/manadv.c
@@ -42,7 +42,7 @@
int manadv_init_obj(struct manadv_obj *obj, uint64_t obj_type)
{
- if (obj_type & ~(MANADV_OBJ_QP | MANADV_OBJ_CQ | MANADV_OBJ_RWQ))
+ if (obj_type & ~(MANADV_OBJ_QP | MANADV_OBJ_CQ | MANADV_OBJ_RWQ | MANADV_OBJ_PD))
return EINVAL;
if (obj_type & MANADV_OBJ_QP) {
@@ -84,5 +84,17 @@
obj->rwq.out->db_page = ctx->db_page;
}
+ if (obj_type & MANADV_OBJ_PD) {
+ struct ibv_pd *ibpd = obj->pd.in;
+ struct mana_pd *pd = container_of(ibpd, struct mana_pd, ibv_pd);
+
+ obj->pd.out->pdn = pd->pdn;
+ }
+
return 0;
}
+
+struct ibv_pd *manadv_alloc_pd(struct ibv_context *context, uint32_t flags)
+{
+ return mana_alloc_pd_ex(context, flags);
+}
diff --git a/providers/mana/manadv.h b/providers/mana/manadv.h
index 27c8fe9..87f6e8f 100644
--- a/providers/mana/manadv.h
+++ b/providers/mana/manadv.h
@@ -52,6 +52,10 @@
void *db_page;
};
+struct manadv_pd {
+ uint32_t pdn;
+};
+
struct manadv_obj {
struct {
struct ibv_qp *in;
@@ -67,16 +71,27 @@
struct ibv_wq *in;
struct manadv_rwq *out;
} rwq;
+ struct {
+ struct ibv_pd *in;
+ struct manadv_pd *out;
+ } pd;
};
enum manadv_obj_type {
MANADV_OBJ_QP = 1 << 0,
MANADV_OBJ_CQ = 1 << 1,
MANADV_OBJ_RWQ = 1 << 2,
+ MANADV_OBJ_PD = 1 << 3,
+};
+
+enum {
+ MANADV_PD_FLAGS_SHORT_PDN = 1 << 0,
};
int manadv_init_obj(struct manadv_obj *obj, uint64_t obj_type);
+struct ibv_pd *manadv_alloc_pd(struct ibv_context *context, uint32_t flags);
+
#ifdef __cplusplus
}
#endif
diff --git a/providers/mana/qp.c b/providers/mana/qp.c
index b6a9a7e..752959b 100644
--- a/providers/mana/qp.c
+++ b/providers/mana/qp.c
@@ -31,6 +31,9 @@
DECLARE_DRV_CMD(mana_create_rc_qp, IB_USER_VERBS_CMD_CREATE_QP,
mana_ib_create_rc_qp, mana_ib_create_rc_qp_resp);
+DECLARE_DRV_CMD(mana_create_uc_qp, IB_USER_VERBS_CMD_CREATE_QP,
+ mana_ib_create_uc_qp, mana_ib_create_uc_qp_resp);
+
static struct ibv_qp *mana_create_qp_raw(struct ibv_pd *ibpd,
struct ibv_qp_init_attr *attr)
{
@@ -216,46 +219,142 @@
uint32_t size = 0;
uint32_t sges = 0;
- if (attr->qp_type == IBV_QPT_RC) {
- switch (type) {
- case USER_RNIC_SEND_QUEUE_REQUESTER:
- /* WQE must have at least one SGE */
- /* For write with imm we need one extra SGE */
- sges = max(1U, attr->cap.max_send_sge) + 1;
- size = attr->cap.max_send_wr * get_large_wqe_size(sges);
- break;
- case USER_RNIC_SEND_QUEUE_RESPONDER:
- size = MANA_PAGE_SIZE;
- break;
- case USER_RNIC_RECV_QUEUE_REQUESTER:
- size = MANA_PAGE_SIZE;
- break;
- case USER_RNIC_RECV_QUEUE_RESPONDER:
- /* WQE must have at least one SGE */
- sges = max(1U, attr->cap.max_recv_sge);
- size = attr->cap.max_recv_wr * get_wqe_size(sges);
- break;
- default:
- return 0;
- }
+ switch (type) {
+ case USER_RNIC_SEND_QUEUE_REQUESTER:
+ /* WQE must have at least one SGE */
+ /* For write with imm we need one extra SGE */
+ sges = max(1U, attr->cap.max_send_sge) + 1;
+ size = align_hw_size(attr->cap.max_send_wr * get_large_wqe_size(sges));
+ break;
+ case USER_RNIC_SEND_QUEUE_RESPONDER:
+ if (attr->qp_type == IBV_QPT_RC)
+ size = align_hw_size(MANA_PAGE_SIZE);
+ break;
+ case USER_RNIC_RECV_QUEUE_REQUESTER:
+ if (attr->qp_type == IBV_QPT_RC)
+ size = align_hw_size(MANA_PAGE_SIZE);
+ break;
+ case USER_RNIC_RECV_QUEUE_RESPONDER:
+ /* WQE must have at least one SGE */
+ sges = max(1U, attr->cap.max_recv_sge);
+ size = align_hw_size(attr->cap.max_recv_wr * get_wqe_size(sges));
+ break;
+ case USER_RNIC_SEND_QUEUE_MM:
+ sges = 2;
+ size = align_hw_size(attr->cap.max_send_wr * get_large_wqe_size(sges));
+ break;
+ default:
+ return 0;
}
- size = align_hw_size(size);
-
if (attr->qp_type == IBV_QPT_RC && type == USER_RNIC_SEND_QUEUE_REQUESTER)
size += sizeof(struct mana_ib_rollback_shared_mem);
return size;
}
-static struct ibv_qp *mana_create_qp_rnic(struct ibv_pd *ibpd,
- struct ibv_qp_init_attr *attr)
+static int mana_create_cmd_qp_rc(struct mana_qp *qp, struct ibv_pd *ibpd,
+ struct ibv_qp_init_attr *attr)
{
- struct mana_context *ctx = to_mctx(ibpd->context);
struct mana_ib_create_rc_qp_resp *qp_resp_drv;
struct mana_create_rc_qp_resp qp_resp = {};
struct mana_ib_create_rc_qp *qp_cmd_drv;
struct mana_create_rc_qp qp_cmd = {};
+ int ret, i;
+
+ qp_cmd_drv = &qp_cmd.drv_payload;
+ qp_resp_drv = &qp_resp.drv_payload;
+
+ for (i = 0; i < USER_RNIC_QUEUE_TYPE_MAX; ++i) {
+ if (i == USER_RNIC_SEND_QUEUE_MM)
+ continue;
+ qp_cmd_drv->queue_buf[i] = (uintptr_t)qp->rnic_qp.queues[i].buffer;
+ qp_cmd_drv->queue_size[i] = qp->rnic_qp.queues[i].size;
+ }
+
+ ret = ibv_cmd_create_qp(ibpd, &qp->ibqp.qp, attr, &qp_cmd.ibv_cmd,
+ sizeof(qp_cmd), &qp_resp.ibv_resp,
+ sizeof(qp_resp));
+ if (ret) {
+ verbs_err(verbs_get_ctx(ibpd->context), "Create QP failed\n");
+ return ret;
+ }
+
+ for (i = 0; i < USER_RNIC_QUEUE_TYPE_MAX; ++i) {
+ if (i == USER_RNIC_SEND_QUEUE_MM)
+ continue;
+ qp->rnic_qp.queues[i].id = qp_resp_drv->queue_id[i];
+ }
+
+ return 0;
+}
+
+enum {
+ MANA_UC_UDATA_SQR = 0,
+ MANA_UC_UDATA_RQR = 1,
+ MANA_UC_UDATA_SMQ = 2,
+};
+
+static int mana_create_cmd_qp_uc(struct mana_qp *qp, struct ibv_pd *ibpd,
+ struct ibv_qp_init_attr *attr)
+{
+ struct mana_ib_create_uc_qp_resp *qp_resp_drv;
+ struct mana_create_uc_qp_resp qp_resp = {};
+ struct mana_ib_create_uc_qp *qp_cmd_drv;
+ struct mana_create_uc_qp qp_cmd = {};
+ int ret;
+
+ qp_cmd_drv = &qp_cmd.drv_payload;
+ qp_resp_drv = &qp_resp.drv_payload;
+
+ qp_cmd_drv->queue_buf[MANA_UC_UDATA_SQR] =
+ (uintptr_t)qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER].buffer;
+ qp_cmd_drv->queue_size[MANA_UC_UDATA_SQR] =
+ qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER].size;
+
+ qp_cmd_drv->queue_buf[MANA_UC_UDATA_RQR] =
+ (uintptr_t)qp->rnic_qp.queues[USER_RNIC_RECV_QUEUE_RESPONDER].buffer;
+ qp_cmd_drv->queue_size[MANA_UC_UDATA_RQR] =
+ qp->rnic_qp.queues[USER_RNIC_RECV_QUEUE_RESPONDER].size;
+
+ qp_cmd_drv->queue_buf[MANA_UC_UDATA_SMQ] =
+ (uintptr_t)qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].buffer;
+ qp_cmd_drv->queue_size[MANA_UC_UDATA_SMQ] =
+ qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].size;
+
+ ret = ibv_cmd_create_qp(ibpd, &qp->ibqp.qp, attr, &qp_cmd.ibv_cmd,
+ sizeof(qp_cmd), &qp_resp.ibv_resp,
+ sizeof(qp_resp));
+ if (ret) {
+ verbs_err(verbs_get_ctx(ibpd->context), "Create QP failed\n");
+ return ret;
+ }
+
+ qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER].id =
+ qp_resp_drv->queue_id[MANA_UC_UDATA_SQR];
+ qp->rnic_qp.queues[USER_RNIC_RECV_QUEUE_RESPONDER].id =
+ qp_resp_drv->queue_id[MANA_UC_UDATA_RQR];
+ qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].id =
+ qp_resp_drv->queue_id[MANA_UC_UDATA_SMQ];
+
+ return 0;
+}
+
+static int mana_create_cmd_qp(struct mana_qp *qp, struct ibv_pd *ibpd,
+ struct ibv_qp_init_attr *attr)
+{
+ if (attr->qp_type == IBV_QPT_RC)
+ return mana_create_cmd_qp_rc(qp, ibpd, attr);
+ else if (attr->qp_type == IBV_QPT_UC)
+ return mana_create_cmd_qp_uc(qp, ibpd, attr);
+ else
+ return -EOPNOTSUPP;
+}
+
+static struct ibv_qp *mana_create_qp_rnic(struct ibv_pd *ibpd,
+ struct ibv_qp_init_attr *attr)
+{
+ struct mana_context *ctx = to_mctx(ibpd->context);
struct mana_qp *qp;
int ret, i;
@@ -263,9 +362,6 @@
if (!qp)
return NULL;
- qp_cmd_drv = &qp_cmd.drv_payload;
- qp_resp_drv = &qp_resp.drv_payload;
-
pthread_spin_init(&qp->sq_lock, PTHREAD_PROCESS_PRIVATE);
pthread_spin_init(&qp->rq_lock, PTHREAD_PROCESS_PRIVATE);
qp->sq_sig_all = attr->sq_sig_all;
@@ -291,29 +387,18 @@
if (qp->rnic_qp.queues[i].size != 0 && !qp->rnic_qp.queues[i].buffer) {
verbs_err(verbs_get_ctx(ibpd->context),
- "Failed to allocate memory for RC queue %d\n", i);
+ "Failed to allocate memory for queue %d\n", i);
errno = ENOMEM;
goto destroy_queues;
}
-
- qp_cmd_drv->queue_buf[i] = (uintptr_t)qp->rnic_qp.queues[i].buffer;
- qp_cmd_drv->queue_size[i] = qp->rnic_qp.queues[i].size;
}
- mana_ib_init_rb_shmem(qp);
-
- ret = ibv_cmd_create_qp(ibpd, &qp->ibqp.qp, attr, &qp_cmd.ibv_cmd,
- sizeof(qp_cmd), &qp_resp.ibv_resp,
- sizeof(qp_resp));
+ ret = mana_create_cmd_qp(qp, ibpd, attr);
if (ret) {
- verbs_err(verbs_get_ctx(ibpd->context), "Create QP failed\n");
errno = ret;
- goto free_rb;
+ goto destroy_queues;
}
- for (i = 0; i < USER_RNIC_QUEUE_TYPE_MAX; ++i)
- qp->rnic_qp.queues[i].id = qp_resp_drv->queue_id[i];
-
qp->ibqp.qp.qp_num = qp->rnic_qp.queues[USER_RNIC_RECV_QUEUE_RESPONDER].id;
ret = mana_store_qp(ctx, qp);
@@ -322,12 +407,12 @@
goto destroy_qp;
}
+ mana_ib_init_rb_shmem(qp);
+
return &qp->ibqp.qp;
destroy_qp:
ibv_cmd_destroy_qp(&qp->ibqp.qp);
-free_rb:
- mana_ib_deinit_rb_shmem(qp);
destroy_queues:
while (i-- > 0)
mana_dealloc_mem(qp->rnic_qp.queues[i].buffer, qp->rnic_qp.queues[i].size);
@@ -346,6 +431,7 @@
case IBV_QPT_RAW_PACKET:
return mana_create_qp_raw(ibpd, attr);
case IBV_QPT_RC:
+ case IBV_QPT_UC:
return mana_create_qp_rnic(ibpd, attr);
default:
verbs_err(verbs_get_ctx(ibpd->context),
@@ -382,7 +468,8 @@
if (attr_mask & IBV_QP_SQ_PSN) {
qp->sq_ssn = 1;
qp->sq_psn = attr->sq_psn;
- gdma_arm_normal_cqe(mana_ib_get_rreq(qp), attr->sq_psn);
+ if (qp->ibqp.qp.qp_type == IBV_QPT_RC)
+ gdma_arm_normal_cqe(mana_ib_get_rreq(qp), attr->sq_psn);
}
break;
default:
@@ -397,7 +484,7 @@
struct ibv_modify_qp cmd = {};
int err;
- if (ibqp->qp_type != IBV_QPT_RC)
+ if (ibqp->qp_type != IBV_QPT_RC && ibqp->qp_type != IBV_QPT_UC)
return EOPNOTSUPP;
pthread_spin_lock(&qp->sq_lock);
@@ -443,7 +530,7 @@
struct mana_context *ctx = to_mctx(ibqp->context);
int ret, i;
- if (ibqp->qp_type == IBV_QPT_RC) {
+ if (ibqp->qp_type == IBV_QPT_RC || ibqp->qp_type == IBV_QPT_UC) {
mana_remove_qp(ctx, qp);
mana_drain_cqes(qp);
}
@@ -459,6 +546,7 @@
ctx->extern_alloc.free(qp->raw_qp.send_buf, ctx->extern_alloc.data);
break;
case IBV_QPT_RC:
+ case IBV_QPT_UC:
pthread_spin_destroy(&qp->sq_lock);
pthread_spin_destroy(&qp->rq_lock);
destroy_shadow_queue(&qp->shadow_sq);
diff --git a/providers/mana/rollback.h b/providers/mana/rollback.h
index 59621a5..c908f3f 100644
--- a/providers/mana/rollback.h
+++ b/providers/mana/rollback.h
@@ -39,6 +39,8 @@
static inline void mana_ib_init_rb_shmem(struct mana_qp *qp)
{
+ if (qp->ibqp.qp.qp_type != IBV_QPT_RC)
+ return;
// take some bytes for rollback memory
struct mana_gdma_queue *req_sq =
&qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER];
@@ -54,6 +56,8 @@
static inline void mana_ib_deinit_rb_shmem(struct mana_qp *qp)
{
+ if (qp->ibqp.qp.qp_type != IBV_QPT_RC)
+ return;
// return back bytes for rollback memory
struct mana_gdma_queue *req_sq =
&qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER];
@@ -62,6 +66,9 @@
static inline void mana_ib_reset_rb_shmem(struct mana_qp *qp)
{
+ if (qp->ibqp.qp.qp_type != IBV_QPT_RC)
+ return;
+
struct mana_ib_rollback_shared_mem *rb_shmem =
mana_ib_get_rollback_sh_mem(qp);
@@ -71,6 +78,9 @@
static inline void mana_ib_update_shared_mem_right_offset(struct mana_qp *qp, uint32_t offset_in_bu)
{
+ if (qp->ibqp.qp.qp_type != IBV_QPT_RC)
+ return;
+
struct mana_ib_rollback_shared_mem *rb_shmem =
mana_ib_get_rollback_sh_mem(qp);
diff --git a/providers/mana/wr.c b/providers/mana/wr.c
index 88ee8ad..e4d7c38 100644
--- a/providers/mana/wr.c
+++ b/providers/mana/wr.c
@@ -191,6 +191,7 @@
{
switch (ibqp->qp_type) {
case IBV_QPT_RC:
+ case IBV_QPT_UC:
return mana_ib_post_recv(ibqp, wr, bad);
default:
verbs_err(verbs_get_ctx(ibqp->context), "QPT not supported %d\n", ibqp->qp_type);
@@ -350,7 +351,7 @@
&send_oob, oob_sge, num_sge, MTU_SIZE(qp->mtu), flags, &gdma_wqe);
if (ret) {
verbs_err(verbs_get_ctx(qp->ibqp.qp.context),
- "rc post send error, ret %d\n", ret);
+ "post send error, ret %d\n", ret);
goto cleanup;
}
@@ -430,6 +431,7 @@
{
switch (ibqp->qp_type) {
case IBV_QPT_RC:
+ case IBV_QPT_UC:
return mana_ib_post_send(ibqp, wr, bad);
default:
verbs_err(verbs_get_ctx(ibqp->context), "QPT not supported %d\n", ibqp->qp_type);
diff --git a/providers/mlx5/cq.c b/providers/mlx5/cq.c
index eeaf4e6..f892658 100644
--- a/providers/mlx5/cq.c
+++ b/providers/mlx5/cq.c
@@ -1842,7 +1842,8 @@
* that match our QP by copying older entries on top of them.
*/
cqe_version = (to_mctx(cq->verbs_cq.cq.context))->cqe_version;
- while ((int) --prod_index - (int) cq->cons_index >= 0) {
+ while (prod_index != cq->cons_index) {
+ --prod_index;
cqe = get_cqe(cq, prod_index & cq->verbs_cq.cq.cqe);
cqe64 = (cq->cqe_sz == 64) ? cqe : cqe + 64;
if (free_res_cqe(cqe64, rsn, srq, cqe_version)) {
diff --git a/providers/mlx5/dbrec.c b/providers/mlx5/dbrec.c
index c5d1c3f..ae99cd9 100644
--- a/providers/mlx5/dbrec.c
+++ b/providers/mlx5/dbrec.c
@@ -105,8 +105,10 @@
mparent_domain->pd_context, 8, 8,
MLX5DV_RES_TYPE_DBR);
- if (db == IBV_ALLOCATOR_USE_DEFAULT)
+ if (db == IBV_ALLOCATOR_USE_DEFAULT) {
+ db = NULL;
goto default_alloc;
+ }
if (!db)
return NULL;
