bnxt_re/lib: Direct Verbs: Support CQ verbs The following Direct Verb (DV) API has been implemented in this patch, by enhancing the driver specific udata in the existing verb. - bnxt_re_dv_create_cq(): Create a CQ of requested size (cqe). The application must have already associated this memory with an ibv_buf using ibv_alloc_user_buf(). The CQ's umem_handle and umem_offset provided by the application are translated into an address for mapping and passed to the driver. Note: Some applications might want to allocate memory for all CQs in one big chunk and associate that entire memory with a single ibv_buf using ibv_alloc_user_buf(). At the time of creating each individual resource, the application should pass a specific offset/length within that memory. Signed-off-by: Sriharsha Basavapatna <sriharsha.basavapatna@broadcom.com> Co-developed-by: Kalesh AP <kalesh-anakkur.purayil@broadcom.com> Signed-off-by: Kalesh AP <kalesh-anakkur.purayil@broadcom.com> Reviewed-by: Selvin Xavier <selvin.xavier@broadcom.com>
diff --git a/providers/bnxt_re/bnxt_re-abi.h b/providers/bnxt_re/bnxt_re-abi.h index 7fe3930..05a9630 100644 --- a/providers/bnxt_re/bnxt_re-abi.h +++ b/providers/bnxt_re/bnxt_re-abi.h
@@ -47,7 +47,7 @@ DECLARE_DRV_CMD(ubnxt_re_pd, IB_USER_VERBS_CMD_ALLOC_PD, empty, bnxt_re_pd_resp); -DECLARE_DRV_CMD(ubnxt_re_cq, IB_USER_VERBS_CMD_CREATE_CQ, +DECLARE_DRV_CMD(ubnxt_re_cq, IB_USER_VERBS_EX_CMD_CREATE_CQ, bnxt_re_cq_req, bnxt_re_cq_resp); DECLARE_DRV_CMD(ubnxt_re_resize_cq, IB_USER_VERBS_CMD_RESIZE_CQ, bnxt_re_resize_cq_req, empty);
diff --git a/providers/bnxt_re/bnxt_re.map b/providers/bnxt_re/bnxt_re.map index fc0b5ff..381e7f0 100644 --- a/providers/bnxt_re/bnxt_re.map +++ b/providers/bnxt_re/bnxt_re.map
@@ -5,5 +5,6 @@ bnxt_re_dv_alloc_db_region; bnxt_re_dv_free_db_region; bnxt_re_dv_get_default_db_region; + bnxt_re_dv_create_cq; local: *; };
diff --git a/providers/bnxt_re/bnxt_re_dv.h b/providers/bnxt_re/bnxt_re_dv.h index 102830b..f0f5b3c 100644 --- a/providers/bnxt_re/bnxt_re_dv.h +++ b/providers/bnxt_re/bnxt_re_dv.h
@@ -21,12 +21,20 @@ uint64_t *dbr; }; +struct bnxt_re_dv_cq_init_attr { + struct ibv_buf *umem_handle; /* handle from ibv_alloc_user_buf() */ + uint64_t umem_offset; /* offset into umem_handle */ + uint32_t ncqe; /* num cq entries */ +}; + struct bnxt_re_dv_db_region_attr * bnxt_re_dv_alloc_db_region(struct ibv_context *ctx); int bnxt_re_dv_free_db_region(struct ibv_context *ctx, struct bnxt_re_dv_db_region_attr *attr); int bnxt_re_dv_get_default_db_region(struct ibv_context *ibvctx, struct bnxt_re_dv_db_region_attr *out); +struct ibv_cq *bnxt_re_dv_create_cq(struct ibv_context *ibvctx, + struct bnxt_re_dv_cq_init_attr *cq_attr); #ifdef __cplusplus } #endif
diff --git a/providers/bnxt_re/dv.c b/providers/bnxt_re/dv.c index 890dc12..6dd1919 100644 --- a/providers/bnxt_re/dv.c +++ b/providers/bnxt_re/dv.c
@@ -108,3 +108,98 @@ return out; } + +static bool bnxt_re_dv_is_valid_buf(struct bnxt_re_dev *dev, + struct ibv_buf *buf, + uint64_t offset, uint32_t size) +{ + return ((offset == align(offset, dev->pg_size)) && + (offset + size <= buf->size)); +} + +static int bnxt_re_dv_create_cq_cmd(struct bnxt_re_dev *dev, + struct ibv_context *ibvctx, + struct bnxt_re_cq *cq, + struct bnxt_re_dv_cq_init_attr *cq_attr, + uint64_t comp_mask, + struct ubnxt_re_cq_resp *resp) +{ + struct bnxt_re_context *cntx = to_bnxt_re_context(ibvctx); + DECLARE_COMMAND_BUFFER(driver_attrs, UVERBS_OBJECT_CQ, + UVERBS_METHOD_CQ_CREATE, 1); + struct ib_uverbs_buffer_desc cq_buf_umem_desc; + struct ibv_cq_init_attr_ex cq_attr_ex = {}; + uint64_t offset = cq_attr->umem_offset; + struct ibv_buf *cq_buf = cq->cq_buf; + struct ubnxt_re_cq cmd = {}; + void *buf_addr; + uint32_t size; + int ret; + + size = cq_attr->ncqe * bnxt_re_get_cqe_sz(); + if (!bnxt_re_dv_is_valid_buf(dev, cq_buf, offset, size)) + return -EINVAL; + + cmd.cq_handle = (uintptr_t)cq; + cmd.comp_mask = comp_mask; + + buf_addr = (uint8_t *)cq_buf->addr + offset; + cmd.cq_va = (uintptr_t)buf_addr; + fill_attr_in_buf_umem(driver_attrs, UVERBS_ATTR_CREATE_CQ_BUF_UMEM, + &cq_buf_umem_desc, cq_buf, + buf_addr, size); + + cq_attr_ex.cqe = cq_attr->ncqe; + cq_attr_ex.comp_mask = 0; + cq_attr_ex.flags = 0; + + memset(resp, 0, sizeof(*resp)); + ret = ibv_cmd_create_cq_ex2(ibvctx, &cq_attr_ex, NULL, + &cq->verbs_cq, + &cmd.ibv_cmd, sizeof(cmd), + &resp->ibv_resp, + sizeof(*resp), 0, driver_attrs); + if (ret) + return ret; + + cq->cqid = resp->cqid; + cq->phase = resp->phase; + cq->udpi = &cntx->udpi; + cq->cntx = cntx; + cq->rand.seed = cq->cqid; + + return 0; +} + +struct ibv_cq *bnxt_re_dv_create_cq(struct ibv_context *ibvctx, + struct bnxt_re_dv_cq_init_attr *cq_attr) +{ + struct bnxt_re_dev *dev = to_bnxt_re_dev(ibvctx->device); + uint64_t comp_mask = BNXT_RE_CQ_FIXED_NUM_CQE_ENABLE; + struct ibv_buf *cq_buf = cq_attr->umem_handle; + struct ubnxt_re_cq_resp resp = {}; + struct bnxt_re_cq *cq; + int ret; + + if (!(dev->vdev.core_support & IB_UVERBS_CORE_SUPPORT_ROBUST_UDATA)) + return NULL; + + if (cq_attr->ncqe > dev->max_cq_depth) + return NULL; + + cq = calloc(1, (sizeof(*cq))); + if (!cq) + return NULL; + + cq->cq_buf = cq_buf; + ret = bnxt_re_dv_create_cq_cmd(dev, ibvctx, cq, cq_attr, comp_mask, &resp); + if (ret) + goto fail; + + cq->dv_cq_flags |= BNXT_DV_CQ_FLAGS_VALID; + return &cq->verbs_cq.cq; + +fail: + free(cq); + return NULL; +}
diff --git a/providers/bnxt_re/main.h b/providers/bnxt_re/main.h index d621d57..cab6beb 100644 --- a/providers/bnxt_re/main.h +++ b/providers/bnxt_re/main.h
@@ -90,6 +90,11 @@ uint32_t pdid; }; +enum bnxt_dv_cq_flags { + BNXT_DV_CQ_FLAGS_NONE = 0, + BNXT_DV_CQ_FLAGS_VALID = 0x1, +}; + struct bnxt_re_cq { struct verbs_cq verbs_cq; uint32_t cqid; @@ -110,6 +115,8 @@ uint8_t resize_tog; bool deffered_db_sup; uint32_t hw_cqes; + struct ibv_buf *cq_buf; + int dv_cq_flags; }; struct bnxt_re_push_buffer {
diff --git a/providers/bnxt_re/memory.c b/providers/bnxt_re/memory.c index 30d972f..79e973a 100644 --- a/providers/bnxt_re/memory.c +++ b/providers/bnxt_re/memory.c
@@ -46,6 +46,13 @@ void bnxt_re_free_mem(struct bnxt_re_mem *mem) { + /* When a DV app allocates its own memory for objects, + * there is no bnxt_re_mem object associated. So just + * return when called from its destroy(). + */ + if (!mem) + return; + if (mem->va_head) { ibv_dofork_range(mem->va_head, mem->size); munmap(mem->va_head, mem->size);
diff --git a/providers/bnxt_re/memory.h b/providers/bnxt_re/memory.h index a0089ee..1a6858b 100644 --- a/providers/bnxt_re/memory.h +++ b/providers/bnxt_re/memory.h
@@ -97,7 +97,6 @@ uint32_t old_head; }; - /* Basic queue operation */ static inline void *bnxt_re_get_hwqe(struct bnxt_re_queue *que, uint32_t idx) {
diff --git a/providers/bnxt_re/verbs.c b/providers/bnxt_re/verbs.c index 1b88abc..d4c8fc1 100644 --- a/providers/bnxt_re/verbs.c +++ b/providers/bnxt_re/verbs.c
@@ -54,6 +54,7 @@ #include "main.h" #include "verbs.h" +#include "bnxt_re_dv.h" static int bnxt_re_poll_one(struct bnxt_re_cq *cq, int nwc, struct ibv_wc *wc, uint32_t *resize); @@ -338,10 +339,11 @@ struct ibv_cq *bnxt_re_create_cq(struct ibv_context *ibvctx, int ncqe, struct ibv_comp_channel *channel, int vec) { - struct bnxt_re_cq *cq; - struct ubnxt_re_cq cmd = {}; - struct ubnxt_re_cq_resp resp; + struct ibv_cq_init_attr_ex cq_attr_ex = {}; struct bnxt_re_mmap_info minfo = {}; + struct ubnxt_re_cq_resp resp = {}; + struct ubnxt_re_cq cmd = {}; + struct bnxt_re_cq *cq; int ret; struct bnxt_re_context *cntx = to_bnxt_re_context(ibvctx); @@ -383,10 +385,15 @@ cmd.cq_va = (uintptr_t)cq->cqq->va; cmd.cq_handle = (uintptr_t)cq; - memset(&resp, 0, sizeof(resp)); - if (ibv_cmd_create_cq(ibvctx, ncqe, channel, vec, - &cq->verbs_cq.cq, &cmd.ibv_cmd, sizeof(cmd), - &resp.ibv_resp, sizeof(resp))) + cq_attr_ex.cqe = ncqe; + cq_attr_ex.channel = channel; + cq_attr_ex.comp_vector = vec; + if (ibv_cmd_create_cq_ex(ibvctx, &cq_attr_ex, NULL, + &cq->verbs_cq, + &cmd.ibv_cmd, + sizeof(cmd), + &resp.ibv_resp, + sizeof(resp), 0)) goto cmdfail; cq->cqid = resp.cqid; @@ -1106,7 +1113,7 @@ return dqed; } -static void bnxt_re_cleanup_cq(struct bnxt_re_qp *qp, struct bnxt_re_cq *cq) +void bnxt_re_cleanup_cq(struct bnxt_re_qp *qp, struct bnxt_re_cq *cq) { struct bnxt_re_queue *que = cq->cqq; struct bnxt_re_bcqe *hdr; @@ -1115,6 +1122,9 @@ void *cqe; int indx, type; + if (cq->dv_cq_flags & BNXT_DV_CQ_FLAGS_VALID) + return; + pthread_spin_lock(&que->qlock); for (indx = 0; indx < que->depth; indx++) { cqe = que->va + indx * bnxt_re_get_cqe_sz();
diff --git a/providers/bnxt_re/verbs.h b/providers/bnxt_re/verbs.h index a04e9ba..3bf108b 100644 --- a/providers/bnxt_re/verbs.h +++ b/providers/bnxt_re/verbs.h
@@ -127,6 +127,8 @@ struct ibv_async_event *event); int bnxt_re_modify_qp_rate_limit(struct ibv_qp *qp, struct ibv_qp_rate_limit_attr *attr); +void bnxt_re_cleanup_cq(struct bnxt_re_qp *qp, + struct bnxt_re_cq *cq); static inline __le64 bnxt_re_update_msn_tbl(uint32_t st_idx, uint32_t npsn, uint32_t start_psn) {