Merge pull request #1848 from zzwzds/docs-remove-stale-and-typo docs: remove stale API references and fix symbol typos in man pages
diff --git a/CMakeLists.txt b/CMakeLists.txt index 0071766..ed2a9c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt
@@ -231,7 +231,11 @@ FIND_PACKAGE(PythonLibs ${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR} EXACT) elseif (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.12") set(Python_EXECUTABLE ${PYTHON_EXECUTABLE}) - FIND_PACKAGE(Python 3 REQUIRED COMPONENTS Interpreter OPTIONAL_COMPONENTS Development) + if(NO_PYVERBS) + FIND_PACKAGE(Python 3 REQUIRED COMPONENTS Interpreter) + else() + FIND_PACKAGE(Python 3 REQUIRED COMPONENTS Interpreter OPTIONAL_COMPONENTS Development) + endif() set(PYTHON_EXECUTABLE ${Python_EXECUTABLE}) if(Python_Development_FOUND) set(PYTHONLIBS_FOUND ${Python_Development_FOUND})
diff --git a/debian/ibverbs-providers.symbols b/debian/ibverbs-providers.symbols index bb74b34..d193e0b 100644 --- a/debian/ibverbs-providers.symbols +++ b/debian/ibverbs-providers.symbols
@@ -232,3 +232,5 @@ ionic_dv_pd_set_udma_mask@IONIC_1.0 59 ionic_dv_pd_set_sqcmb@IONIC_1.0 59 ionic_dv_pd_set_rqcmb@IONIC_1.0 59 + IONIC_1.1@IONIC_1.1 66 + ionic_dv_create_qp_ex@IONIC_1.1 66
diff --git a/kernel-headers/rdma/ionic-abi.h b/kernel-headers/rdma/ionic-abi.h index 2c70ac1..3d8d569 100644 --- a/kernel-headers/rdma/ionic-abi.h +++ b/kernel-headers/rdma/ionic-abi.h
@@ -22,6 +22,10 @@ #define IONIC_CMB_WC 8 #define IONIC_CMB_UC 16 +enum ionic_ctx_resp_comp_mask { + IONIC_CTX_CMASK_IONIC_FLAGS = 1 << 0, +}; + struct ionic_ctx_req { __u32 rsvd[2]; }; @@ -46,8 +50,9 @@ __u8 udma_count; __u8 expdb_mask; __u8 expdb_qtypes; + __u8 rcq_sign_bit; - __u8 rsvd2[3]; + __u16 comp_mask; __aligned_u64 phc_offset; }; @@ -85,6 +90,8 @@ __u8 rq_cmb; __u8 udma_mask; __u8 rsvd[3]; + __u32 ionic_flags; + __u32 rsvd_pad; }; struct ionic_qp_resp {
diff --git a/libibverbs/driver.h b/libibverbs/driver.h index 711450c..3507999 100644 --- a/libibverbs/driver.h +++ b/libibverbs/driver.h
@@ -873,32 +873,53 @@ } /** - * In case the length is 0, the buffer size is used. Handles both - * dmabuf-backed and plain user virtual address buffers. + * In case the length is 0, the buffer size is used. */ static inline void fill_attr_in_buf_umem(struct ibv_command_buffer *cmdb, uint16_t attr_id, struct ib_uverbs_buffer_desc *storage, struct ibv_buf *buf, void *addr, size_t length) { + if (!buf || buf->dmabuf_fd == -1) + return; + + *storage = (struct ib_uverbs_buffer_desc){ + .type = IB_UVERBS_BUFFER_TYPE_DMABUF, + .fd = buf->dmabuf_fd, + .addr = addr ? (uintptr_t)addr - (uintptr_t)buf->addr : 0, + .length = length ? length : buf->size, + }; + fill_attr_in_ptr(cmdb, attr_id, storage); + cmdb->fallback_ioctl_only = 1; +} + +/** + * Like fill_attr_in_buf_umem(), but also describes plain user virtual + * address buffers (IB_UVERBS_BUFFER_TYPE_VA). The attribute is always added + * and is mandatory, so only use it where the caller requires UMEM attribute + * support from the kernel (e.g. direct verbs), never on generic paths. + * In case the length is 0, the buffer size is used. + */ +static inline void +fill_attr_in_buf_umem_va(struct ibv_command_buffer *cmdb, uint16_t attr_id, + struct ib_uverbs_buffer_desc *storage, + struct ibv_buf *buf, void *addr, size_t length) +{ if (!buf) return; if (buf->dmabuf_fd != -1) { - *storage = (struct ib_uverbs_buffer_desc){ - .type = IB_UVERBS_BUFFER_TYPE_DMABUF, - .fd = buf->dmabuf_fd, - .addr = addr ? (uintptr_t)addr - (uintptr_t)buf->addr : 0, - .length = length ? length : buf->size, - }; - } else { - *storage = (struct ib_uverbs_buffer_desc){ - .type = IB_UVERBS_BUFFER_TYPE_VA, - .fd = -1, - .addr = (uintptr_t)(addr ? addr : buf->addr), - .length = length ? length : buf->size, - }; + fill_attr_in_buf_umem(cmdb, attr_id, storage, buf, addr, + length); + return; } + + *storage = (struct ib_uverbs_buffer_desc){ + .type = IB_UVERBS_BUFFER_TYPE_VA, + .fd = -1, + .addr = (uintptr_t)(addr ? addr : buf->addr), + .length = length ? length : buf->size, + }; fill_attr_in_ptr(cmdb, attr_id, storage); cmdb->fallback_ioctl_only = 1; }
diff --git a/providers/bnxt_re/dv.c b/providers/bnxt_re/dv.c index b165a55..5be9a62 100644 --- a/providers/bnxt_re/dv.c +++ b/providers/bnxt_re/dv.c
@@ -145,9 +145,9 @@ buf_addr = (uint8_t *)cq_buf->addr + offset; cmd.cq_va = (uintptr_t)buf_addr; - fill_attr_in_buf_umem(driver_attrs, UVERBS_ATTR_CREATE_CQ_BUF_UMEM, - &cq_buf_umem_desc, cq_buf, - buf_addr, size); + fill_attr_in_buf_umem_va(driver_attrs, UVERBS_ATTR_CREATE_CQ_BUF_UMEM, + &cq_buf_umem_desc, cq_buf, + buf_addr, size); cq_attr_ex.cqe = cq_attr->ncqe; cq_attr_ex.comp_mask = 0; @@ -304,9 +304,9 @@ req.sq_slots = dv_qp_attr->sq_slots; req.sq_npsn = dv_qp_attr->sq_npsn; buf_addr = (uint8_t *)sq_buf->addr + offset; - fill_attr_in_buf_umem(driver_attrs, UVERBS_ATTR_CREATE_QP_SQ_BUF_UMEM, - &sq_buf_umem_desc, sq_buf, - buf_addr, size); + fill_attr_in_buf_umem_va(driver_attrs, UVERBS_ATTR_CREATE_QP_SQ_BUF_UMEM, + &sq_buf_umem_desc, sq_buf, + buf_addr, size); /* Keep qpsva as fallback for kernels without SQ_BUF_UMEM support */ req.qpsva = (uintptr_t)buf_addr; @@ -319,10 +319,10 @@ return -EINVAL; buf_addr = (uint8_t *)rq_buf->addr + offset; - fill_attr_in_buf_umem(driver_attrs, - UVERBS_ATTR_CREATE_QP_RQ_BUF_UMEM, - &rq_buf_umem_desc, rq_buf, - buf_addr, size); + fill_attr_in_buf_umem_va(driver_attrs, + UVERBS_ATTR_CREATE_QP_RQ_BUF_UMEM, + &rq_buf_umem_desc, rq_buf, + buf_addr, size); /* Keep qprva as fallback for kernels without RQ_BUF_UMEM support */ req.qprva = (uintptr_t)buf_addr; }
diff --git a/providers/efa/verbs.c b/providers/efa/verbs.c index 25981f5..614542f 100644 --- a/providers/efa/verbs.c +++ b/providers/efa/verbs.c
@@ -481,7 +481,10 @@ return efa_wq_get_next_wrid_idx_locked(wq, wr_id) | wq->shifted_gen; } -static void efa_wq_cqe_finalize(struct efa_wq *wq, struct efa_io_cdesc_common *cqe) +static inline void efa_wq_cqe_finalize(struct efa_wq *wq, + struct efa_io_cdesc_common *cqe) ALWAYS_INLINE; +static inline void efa_wq_cqe_finalize(struct efa_wq *wq, + struct efa_io_cdesc_common *cqe) { if (wq->need_lock) pthread_spin_lock(&wq->wqlock);
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index 20b95b8..acda771 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c
@@ -305,7 +305,7 @@ static void hns_roce_write512(uint64_t *dest, uint64_t *val) { - mmio_memcpy_x64(dest, val, sizeof(struct hns_roce_rc_sq_wqe)); + mmio_memcpy_x64(dest, val, 1 << HNS_ROCE_SQWQE_SHIFT); } static void hns_roce_write_dwqe(struct hns_roce_qp *qp, void *wqe)
diff --git a/providers/ionic/CMakeLists.txt b/providers/ionic/CMakeLists.txt index a28083f..795a100 100644 --- a/providers/ionic/CMakeLists.txt +++ b/providers/ionic/CMakeLists.txt
@@ -1,5 +1,5 @@ rdma_shared_provider(ionic libionic.map - 1 1.0.${PACKAGE_VERSION} + 1 1.1.${PACKAGE_VERSION} ionic.c ionic_verbs.c ionic_memory.c
diff --git a/providers/ionic/ionic.c b/providers/ionic/ionic.c index 6451aab..bd976d4 100644 --- a/providers/ionic/ionic.c +++ b/providers/ionic/ionic.c
@@ -78,6 +78,11 @@ ctx->sq_expdb = !!(resp.expdb_qtypes & IONIC_EXPDB_SQ); ctx->rq_expdb = !!(resp.expdb_qtypes & IONIC_EXPDB_RQ); + ctx->robust_udata = !!(verbs_get_device(ibdev)->core_support & + IB_UVERBS_CORE_SUPPORT_ROBUST_UDATA); + ctx->ionic_flags_supported = !!(resp.comp_mask & IONIC_CTX_CMASK_IONIC_FLAGS); + ctx->rcq_sign_bit = resp.rcq_sign_bit; + mask = (1u << ctx->pg_shift) - 1; ctx->dbpage_page = ionic_map_device(1u << ctx->pg_shift, cmd_fd, resp.dbell_offset & ~mask);
diff --git a/providers/ionic/ionic.h b/providers/ionic/ionic.h index cb1a8da..e49e223 100644 --- a/providers/ionic/ionic.h +++ b/providers/ionic/ionic.h
@@ -22,6 +22,7 @@ #include "ionic_memory.h" #include "ionic_queue.h" #include "ionic_table.h" +#include "ionic_dv.h" #include <stdio.h> #include <inttypes.h> @@ -45,6 +46,17 @@ #define IONIC_PD_TAG_CQ (IONIC_PD_TAG | 1) #define IONIC_PD_TAG_SQ (IONIC_PD_TAG | 2) #define IONIC_PD_TAG_RQ (IONIC_PD_TAG | 3) +#define IONIC_PD_TAG_RCQ (IONIC_PD_TAG | 4) + +#define IONIC_SET_UDATA(ctx, req, field, val) \ + ({ \ + int __rc = 0; \ + if ((ctx)->robust_udata) \ + (req)->field = (val); \ + else \ + __rc = ENOTSUP; \ + __rc; \ + }) enum { IONIC_CQ_SUPPORTED_WC_FLAGS = @@ -83,6 +95,18 @@ IBV_QP_EX_WITH_SEND_WITH_INV, }; +enum { + IONIC_QP_SUPPORTED_INIT_ATTR_MASK = + IONIC_QP_INIT_ATTR_MASK_FLAGS, + + IONIC_QP_SUPPORTED_INIT_ATTR_FLAGS = + IONIC_QP_INIT_ATTR_RCCL | + IONIC_QP_INIT_ATTR_RCCL_DATA | + IONIC_QP_INIT_ATTR_RCCL_RDFENCE | + IONIC_QP_INIT_ATTR_RCCL_RX_OFFLOAD | + IONIC_QP_INIT_ATTR_RCCL_RCQ, +}; + struct ionic_ctx { struct verbs_context vctx; @@ -102,6 +126,9 @@ uint8_t expdb_mask; bool sq_expdb; bool rq_expdb; + bool robust_udata; + bool ionic_flags_supported; + uint8_t rcq_sign_bit; void *dbpage_page; uint64_t *dbpage; @@ -196,6 +223,16 @@ struct ionic_rq_meta { struct ionic_rq_meta *next; uint64_t wrid; + struct { + uint64_t timestamp; + uint32_t seq; + uint32_t imm_rkey; + uint32_t sts_len; + uint8_t valid:1; + uint8_t ready:1; + uint8_t error:1; + uint8_t op; + } rcqe; }; struct ionic_rq { @@ -253,6 +290,7 @@ struct ibv_send_wr ex_wr; int ex_rc; bool sig_all; + bool rcq; }; struct ionic_ah { @@ -374,4 +412,8 @@ /* ionic_verbs.h */ void ionic_verbs_set_ops(struct ionic_ctx *ctx); +struct ibv_qp *ionic_create_qp_ex_common(struct ibv_context *ibctx, + struct ibv_qp_init_attr_ex *ex, + struct ionic_dv_qp_init_attr_ex *ionic_ex); + #endif /* IONIC_H */
diff --git a/providers/ionic/ionic_dv.c b/providers/ionic/ionic_dv.c index 738fcc3..92900dc 100644 --- a/providers/ionic/ionic_dv.c +++ b/providers/ionic/ionic_dv.c
@@ -105,3 +105,15 @@ return 0; } + +struct ibv_qp *ionic_dv_create_qp_ex(struct ibv_context *ibctx, + struct ibv_qp_init_attr_ex *ex, + struct ionic_dv_qp_init_attr_ex *ionic_ex) +{ + if (!is_ionic_ctx(ibctx)) { + errno = EINVAL; + return NULL; + } + + return ionic_create_qp_ex_common(ibctx, ex, ionic_ex); +}
diff --git a/providers/ionic/ionic_dv.h b/providers/ionic/ionic_dv.h index a11f5a2..d8efda8 100644 --- a/providers/ionic/ionic_dv.h +++ b/providers/ionic/ionic_dv.h
@@ -68,6 +68,35 @@ */ int ionic_dv_pd_set_rqcmb(struct ibv_pd *ibpd, bool enable, bool expdb, bool require); +enum ionic_qp_init_attr_mask { + IONIC_QP_INIT_ATTR_MASK_FLAGS = 1 << 0, +}; + +enum ionic_qp_init_attr_flags { + IONIC_QP_INIT_ATTR_RCCL = 1 << 16, + IONIC_QP_INIT_ATTR_RCCL_DATA = 1 << 17, + IONIC_QP_INIT_ATTR_RCCL_RDFENCE = 1 << 18, + IONIC_QP_INIT_ATTR_RCCL_RX_OFFLOAD = 1 << 19, + IONIC_QP_INIT_ATTR_RCCL_RCQ = 1 << 24, + IONIC_QP_INIT_ATTR_RCCL_P2P = 1 << 25, +}; + +struct ionic_dv_qp_init_attr_ex { + uint64_t comp_mask; + uint32_t flags; +}; + +/** + * ionic_dv_create_qp_ex - Create a queue pair with ionic-specific attributes. + * + * @ibctx - Device context. + * @ex - Standard QP init attributes. + * @ionic_ex - Ionic-specific QP init attributes (transport mode, rcq paths). + */ +struct ibv_qp *ionic_dv_create_qp_ex(struct ibv_context *ibctx, + struct ibv_qp_init_attr_ex *ex, + struct ionic_dv_qp_init_attr_ex *ionic_ex); + #ifdef __cplusplus } #endif
diff --git a/providers/ionic/ionic_fw.h b/providers/ionic/ionic_fw.h index 6598213..1d80163 100644 --- a/providers/ionic/ionic_fw.h +++ b/providers/ionic/ionic_fw.h
@@ -217,4 +217,48 @@ return spec; } +static inline uint32_t ionic_v1_rcqe_seq(uint32_t seq_opf) +{ + return seq_opf & IONIC_V1_CQE_RCQE_SEQ_MASK; +} + +static inline uint8_t ionic_v1_rcqe_op(uint32_t seq_opf) +{ + return seq_opf >> IONIC_V1_CQE_RCQE_OP_SHIFT; +} + +static inline bool ionic_v1_rcqe_valid(uint32_t seq_opf) +{ + return seq_opf & IONIC_V1_CQE_RCQE_FLAG_V; +} + +static inline bool ionic_v1_rcqe_ready(uint32_t seq_opf) +{ + return seq_opf & IONIC_V1_CQE_RCQE_FLAG_I; +} + +#define IONIC_RCQ_SIZE 4096 + +struct ionic_rcq_hdr { + __be32 seq; + __be32 ack; +}; + +struct ionic_rcq { + union { + uint8_t bytes[IONIC_RCQ_SIZE]; + struct ionic_rcq_hdr hdr; + }; +}; + +static inline uint32_t ionic_rcq_seq(struct ionic_rcq *rcq) +{ + return be32toh(rcq->hdr.seq) & IONIC_V1_CQE_RCQE_SEQ_MASK; +} + +static inline void ionic_rcq_ack(struct ionic_rcq *rcq, uint32_t ack) +{ + rcq->hdr.ack = htobe32(ack); +} + #endif /* IONIC_FW_H */
diff --git a/providers/ionic/ionic_fw_types.h b/providers/ionic/ionic_fw_types.h index b57f7f9..4653673 100644 --- a/providers/ionic/ionic_fw_types.h +++ b/providers/ionic/ionic_fw_types.h
@@ -70,22 +70,34 @@ __u8 data[32]; }; +struct ionic_v1_cqe_send { + __u8 rsvd[4]; + __be32 msg_msn; + __u8 rsvd2[8]; + __le64 npg_wqe_idx_timestamp; +}; + +struct ionic_v1_cqe_recv { + __le64 wqe_idx_timestamp; + __be32 src_qpn_op; + __u8 src_mac[6]; + __be16 vlan_tag; + __be32 imm_data_rkey; +}; + +struct ionic_v1_cqe_rcqe { + __be64 wqe_idx_timestamp; + __u8 rsvd[8]; + __be32 seq_op_flags; + __be32 imm_data_rkey; +}; + /* completion queue v1 cqe */ struct ionic_v1_cqe { union { - struct { - __le64 wqe_idx_timestamp; - __be32 src_qpn_op; - __u8 src_mac[6]; - __be16 vlan_tag; - __be32 imm_data_rkey; - } recv; - struct { - __u8 rsvd[4]; - __be32 msg_msn; - __u8 rsvd2[8]; - __le64 npg_wqe_idx_timestamp; - } send; + struct ionic_v1_cqe_send send; + struct ionic_v1_cqe_recv recv; + struct ionic_v1_cqe_rcqe rcqe; }; __be32 status_length; __be32 qid_type_flags; @@ -97,6 +109,14 @@ IONIC_V1_CQE_TIMESTAMP_SHIFT = 16, }; +/* bits for rcqe seq_op_flags */ +enum ionic_v1_cqe_rcqe_op_flag_bits { + IONIC_V1_CQE_RCQE_SEQ_MASK = 0xffffff, + IONIC_V1_CQE_RCQE_FLAG_V = 1u << 24, + IONIC_V1_CQE_RCQE_FLAG_I = 1u << 25, + IONIC_V1_CQE_RCQE_OP_SHIFT = 28, +}; + /* bits for cqe recv */ enum ionic_v1_cqe_src_qpn_bits { IONIC_V1_CQE_RECV_QPN_MASK = 0xffffff, @@ -126,7 +146,7 @@ IONIC_V1_CQE_TYPE_RECV = 1, IONIC_V1_CQE_TYPE_SEND_MSN = 2, IONIC_V1_CQE_TYPE_SEND_NPG = 3, - IONIC_V1_CQE_TYPE_RECV_INDIR = 4, + IONIC_V1_CQE_TYPE_RECV_RCQE = 4, }; /* v1 base wqe header */
diff --git a/providers/ionic/ionic_queue.c b/providers/ionic/ionic_queue.c index 28458a6..518a986 100644 --- a/providers/ionic/ionic_queue.c +++ b/providers/ionic/ionic_queue.c
@@ -9,6 +9,7 @@ #include "ionic.h" #include "ionic_queue.h" #include "ionic_memory.h" +#include "ionic_fw.h" static void ionic_queue_map(struct ionic_queue *q, struct ionic_pd *pd, uint64_t pd_tag, int stride) { @@ -65,6 +66,9 @@ q->size = BIT_ULL(q->depth_log2 + q->stride_log2); q->mask = BIT(q->depth_log2) - 1; + if (pd_tag == IONIC_PD_TAG_RCQ) + q->size += IONIC_RCQ_SIZE; + ionic_queue_map(q, pd, pd_tag, stride); if (!q->ptr) return errno;
diff --git a/providers/ionic/ionic_verbs.c b/providers/ionic/ionic_verbs.c index 1aa08f5..2e6e698 100644 --- a/providers/ionic/ionic_verbs.c +++ b/providers/ionic/ionic_verbs.c
@@ -1401,6 +1401,124 @@ } } +static int ionic_validate_rcq_wqe_idx(uint16_t prod, uint16_t cons, + uint16_t idx, uint16_t mask) +{ + return ((idx - cons) & mask) < ((prod - cons) & mask) ? 0 : -EIO; +} + +static void ionic_comp_rcqe(struct ionic_ctx *ctx, + struct ionic_qp *qp, + struct ionic_v1_cqe *cqe) +{ + struct ionic_rq_meta *meta; + uint64_t wqe_idx_timestamp; + uint32_t wqe_idx; + uint32_t seq_opf; + bool error; + int rc; + + seq_opf = be32toh(cqe->rcqe.seq_op_flags); + + if (!ionic_v1_rcqe_valid(seq_opf)) + return; + + wqe_idx_timestamp = be64toh(cqe->rcqe.wqe_idx_timestamp); + wqe_idx = wqe_idx_timestamp & IONIC_V1_CQE_WQE_IDX_MASK; + + if (unlikely(wqe_idx >> qp->rq.queue.depth_log2)) { + verbs_err(&ctx->vctx, "invalid wqe_idx %#x", wqe_idx); + return; + } + + rc = ionic_validate_rcq_wqe_idx(qp->rq.queue.prod, + qp->rq.queue.cons, + wqe_idx, + qp->rq.queue.mask); + if (rc) { + struct ionic_cq *cq = + to_ionic_vcq_cq(qp->vqp.qp.recv_cq, qp->udma_idx); + + verbs_err(&ctx->vctx, "wqe is not posted for rcqe seq %u rcqe idx %u qpid %u prod %u cons %u cqid %u", + ionic_v1_rcqe_seq(seq_opf), wqe_idx, + qp->qpid, qp->rq.queue.prod, qp->rq.queue.cons, cq->cqid); + return; + } + + meta = &qp->rq.meta[qp->rq.meta_idx[wqe_idx]]; + error = ionic_v1_cqe_error(cqe); + + meta->rcqe.valid = true; + meta->rcqe.error = error; + meta->rcqe.ready = ionic_v1_rcqe_ready(seq_opf); + meta->rcqe.op = ionic_v1_rcqe_op(seq_opf); + meta->rcqe.seq = ionic_v1_rcqe_seq(seq_opf); + meta->rcqe.imm_rkey = be32toh(cqe->rcqe.imm_data_rkey); + meta->rcqe.sts_len = be32toh(cqe->status_length); + meta->rcqe.timestamp = wqe_idx_timestamp >> IONIC_V1_CQE_TIMESTAMP_SHIFT; + + verbs_debug(&ctx->vctx, "received rcq_seq %u wqe idx %u", meta->rcqe.seq, wqe_idx); +} + +static int ionic_poll_rcq_recv(struct ionic_ctx *ctx, + struct ionic_cq *cq, + struct ionic_qp *qp, + struct ibv_wc *wc) +{ + struct ionic_v1_cqe cqe = {}; + struct ionic_rq_meta *meta; + uint64_t wqe_idx_timestamp; + struct ionic_rcq *rcq; + uint32_t rcq_seq; + + if (qp->rq.flush) + return 0; + + rcq = qp->rq.queue.ptr + qp->rq.queue.size - IONIC_RCQ_SIZE; + + rcq_seq = ionic_rcq_seq(rcq); + ionic_rcq_ack(rcq, rcq_seq); + + if (ionic_queue_empty(&qp->rq.queue)) + return 0; + + meta = &qp->rq.meta[qp->rq.meta_idx[qp->rq.queue.cons]]; + if (!meta->rcqe.valid) + return 0; + + if (!meta->rcqe.ready && ((rcq_seq - meta->rcqe.seq) & BIT(ctx->rcq_sign_bit))) + return -EAGAIN; + + verbs_debug(&ctx->vctx, "polled rcq_seq %u rcqe_seq %u sign %lu cons %u", + rcq_seq, meta->rcqe.seq, BIT(ctx->rcq_sign_bit), qp->rq.queue.cons); + + wqe_idx_timestamp = qp->rq.queue.cons | + (meta->rcqe.timestamp << IONIC_V1_CQE_TIMESTAMP_SHIFT); + cqe.recv.wqe_idx_timestamp = htole64(wqe_idx_timestamp); + cqe.recv.src_qpn_op = htobe32(meta->rcqe.op << IONIC_V1_CQE_RECV_OP_SHIFT); + cqe.recv.imm_data_rkey = htobe32(meta->rcqe.imm_rkey); + cqe.status_length = htobe32(meta->rcqe.sts_len); + + return ionic_poll_recv(ctx, cq, qp, &cqe, wc); +} + +static int ionic_poll_rcq_recv_many(struct ionic_ctx *ctx, struct ionic_cq *cq, + struct ionic_qp *qp, + struct ibv_wc *wc, int nwc) +{ + int rc = 0, npolled = 0; + + while (npolled < nwc) { + rc = ionic_poll_rcq_recv(ctx, cq, qp, wc + npolled); + if (rc <= 0) + break; + + npolled += rc; + } + + return npolled ?: rc; +} + static int ionic_poll_vcq_cq(struct ionic_ctx *ctx, struct ionic_cq *cq, int nwc, struct ibv_wc *wc) { @@ -1516,7 +1634,9 @@ break; - case IONIC_V1_CQE_TYPE_RECV_INDIR: + case IONIC_V1_CQE_TYPE_RECV_RCQE: + ionic_comp_rcqe(ctx, qp, cqe); + list_del(&qp->cq_poll_rq); list_add_tail(&cq->poll_rq, &qp->cq_poll_rq); break; @@ -1532,6 +1652,26 @@ cq->color = ionic_color_wrap(cq->q.prod, cq->color); } + /* poll out-of-order rcq completions for recv queue */ + list_for_each_safe(&cq->poll_rq, qp, qp_next, cq_poll_rq) { + if (npolled == nwc) + goto out; + + ionic_rq_spin_lock(qp); + rc = ionic_poll_rcq_recv_many(ctx, cq, qp, + wc + npolled, nwc - npolled); + ionic_rq_spin_unlock(qp); + + if (rc == 0) + list_del_init(&qp->cq_poll_rq); + else if (rc == -EAGAIN) + rc = 0; + else if (rc < 0) + goto out; + + npolled += rc; + } + /* lastly, flush send and recv queues */ if (likely(!cq->flush)) goto out; @@ -1726,18 +1866,19 @@ qp->sq.spec = ionic_v1_use_spec_sge(max_sge, ctx->spec); - if (qp->sq.cmb & IONIC_CMB_EXPDB) { - wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data, - qp->sq.spec, true); + wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data, + qp->sq.spec, false); - if (!ionic_expdb_wqe_size_supported(ctx, wqe_size)) + if (qp->sq.cmb & IONIC_CMB_EXPDB) { + uint32_t expdb_size = ionic_v1_send_wqe_min_size(max_sge, max_data, + qp->sq.spec, true); + + if (ionic_expdb_wqe_size_supported(ctx, expdb_size)) + wqe_size = expdb_size; + else qp->sq.cmb &= ~IONIC_CMB_EXPDB; } - if (!(qp->sq.cmb & IONIC_CMB_EXPDB)) - wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data, - qp->sq.spec, false); - rc = ionic_queue_init(&qp->sq.queue, pd, IONIC_PD_TAG_SQ, ctx->pg_shift, max_wr, wqe_size); if (rc) @@ -1798,19 +1939,25 @@ if (max_sge > ionic_v1_recv_wqe_max_sge(ctx->max_stride, 0, false)) return EINVAL; - pd_tag = IONIC_PD_TAG_RQ; + if (qp->rcq) { + pd_tag = IONIC_PD_TAG_RCQ; + qp->rq.cmb = 0; + } else { + pd_tag = IONIC_PD_TAG_RQ; + } qp->rq.spec = ionic_v1_use_spec_sge(max_sge, ctx->spec); - if (qp->rq.cmb & IONIC_CMB_EXPDB) { - wqe_size = ionic_v1_recv_wqe_min_size(max_sge, qp->rq.spec, true); + wqe_size = ionic_v1_recv_wqe_min_size(max_sge, qp->rq.spec, false); - if (!ionic_expdb_wqe_size_supported(ctx, wqe_size)) + if (qp->rq.cmb & IONIC_CMB_EXPDB) { + uint32_t expdb_size = ionic_v1_recv_wqe_min_size(max_sge, qp->rq.spec, true); + + if (ionic_expdb_wqe_size_supported(ctx, expdb_size)) + wqe_size = expdb_size; + else qp->rq.cmb &= ~IONIC_CMB_EXPDB; } - if (!(qp->rq.cmb & IONIC_CMB_EXPDB)) - wqe_size = ionic_v1_recv_wqe_min_size(max_sge, qp->rq.spec, false); - rc = ionic_queue_init(&qp->rq.queue, pd, pd_tag, ctx->pg_shift, max_wr, wqe_size); if (rc) goto err_rq; @@ -2135,17 +2282,45 @@ ionic_wr_set(qp, 0, 0, NULL); } -static struct ibv_qp *ionic_create_qp_ex(struct ibv_context *ibctx, - struct ibv_qp_init_attr_ex *ex) +struct ibv_qp *ionic_create_qp_ex_common(struct ibv_context *ibctx, + struct ibv_qp_init_attr_ex *ex, + struct ionic_dv_qp_init_attr_ex *ionic_ex) { struct ionic_ctx *ctx = to_ionic_ctx(ibctx); struct ionic_pd *pd = to_ionic_pd(ex->pd); - struct uionic_qp_resp resp; - struct uionic_qp req; + struct uionic_qp_resp resp = {}; + struct uionic_qp req = {}; struct ionic_qp *qp; struct ionic_cq *cq; int rc; + if (ionic_ex) { + if (!check_comp_mask(ionic_ex->comp_mask, + IONIC_QP_SUPPORTED_INIT_ATTR_MASK)) { + rc = ENOTSUP; + goto err_qp; + } + + if (ionic_ex->comp_mask & IONIC_QP_INIT_ATTR_MASK_FLAGS) { + if (!ctx->ionic_flags_supported || + !check_comp_mask(ionic_ex->flags, + IONIC_QP_SUPPORTED_INIT_ATTR_FLAGS)) { + rc = ENOTSUP; + goto err_qp; + } + + if ((ionic_ex->flags & IONIC_QP_INIT_ATTR_RCCL_RCQ) && + !ctx->rcq_sign_bit) { + rc = EINVAL; + goto err_qp; + } + + rc = IONIC_SET_UDATA(ctx, &req, ionic_flags, ionic_ex->flags); + if (rc) + goto err_qp; + } + } + if ((ex->comp_mask & IONIC_QP_REQUIRED_COMP_MASK) != IONIC_QP_REQUIRED_COMP_MASK) { rc = ENOTSUP; @@ -2186,6 +2361,7 @@ qp->has_rq = true; qp->lockfree = ex->pd ? ionic_pd_lockfree(ex->pd) : false; qp->sig_all = ex->sq_sig_all; + qp->rcq = ctx->rcq_sign_bit && (req.ionic_flags & IONIC_QP_INIT_ATTR_RCCL_RCQ); list_node_init(&qp->cq_poll_sq); list_node_init(&qp->cq_poll_rq); @@ -2328,6 +2504,12 @@ return NULL; } +static struct ibv_qp *ionic_create_qp_ex(struct ibv_context *ibctx, + struct ibv_qp_init_attr_ex *ex) +{ + return ionic_create_qp_ex_common(ibctx, ex, NULL); +} + static void ionic_flush_qp(struct ionic_ctx *ctx, struct ionic_qp *qp) { struct ionic_cq *cq; @@ -3229,6 +3411,7 @@ wqe->base.wqe_idx = htole64(qp->rq.queue.prod); wqe->base.num_sge_key = wr->num_sge; + memset(&meta->rcqe, 0, sizeof(meta->rcqe)); qp->rq.meta_idx[qp->rq.queue.prod] = meta - qp->rq.meta; @@ -3386,7 +3569,7 @@ struct ibv_ah_attr *attr) { struct ibv_pd *root_ibpd = to_ionic_root_ibpd(ibpd); - struct uionic_ah_resp resp; + struct uionic_ah_resp resp = {}; struct ionic_ah *ah; int rc;
diff --git a/providers/ionic/libionic.map b/providers/ionic/libionic.map index 3efca32..be963d7 100644 --- a/providers/ionic/libionic.map +++ b/providers/ionic/libionic.map
@@ -10,3 +10,8 @@ ionic_dv_pd_set_rqcmb; local: *; }; + +IONIC_1.1 { + global: + ionic_dv_create_qp_ex; +} IONIC_1.0;
diff --git a/providers/ionic/man/CMakeLists.txt b/providers/ionic/man/CMakeLists.txt index 2827560..d2460e5 100644 --- a/providers/ionic/man/CMakeLists.txt +++ b/providers/ionic/man/CMakeLists.txt
@@ -1,5 +1,6 @@ rdma_man_pages( ionicdv.7.md + ionic_dv_create_qp_ex.3.md ionic_dv_ctx_get_udma_count.3.md ionic_dv_ctx_get_udma_mask.3.md ionic_dv_pd_get_udma_mask.3.md
diff --git a/providers/ionic/man/ionic_dv_create_qp_ex.3.md b/providers/ionic/man/ionic_dv_create_qp_ex.3.md new file mode 100644 index 0000000..b8f0883 --- /dev/null +++ b/providers/ionic/man/ionic_dv_create_qp_ex.3.md
@@ -0,0 +1,138 @@ +--- +layout: page +title: IONIC_DV_CREATE_QP_EX +section: 3 +tagline: Verbs +date: 2025-06-23 +header: "Ionic Programmer's Manual" +footer: ionic +--- + +# NAME + +ionic_dv_create_qp_ex - Create a queue pair with ionic-specific attributes + +# SYNOPSIS + +```c +#include <infiniband/ionic_dv.h> + +struct ibv_qp *ionic_dv_create_qp_ex(struct ibv_context *ibctx, + struct ibv_qp_init_attr_ex *ex, + struct ionic_dv_qp_init_attr_ex *ionic_ex); +``` + +# DESCRIPTION + +**ionic_dv_create_qp_ex()** creates a queue pair (QP) with ionic-specific +driver properties. It extends the standard **ibv_create_qp_ex**(3) interface +with ionic-specific attributes for configuring RCCL (RC Collective) +operations and Reorder Completion Queue (RCQ) behavior. + +The caller specifies **IBV_QPT_RC** as the QP type in the standard *ex* +attributes. The returned QP always has *qp_type* == **IBV_QPT_RC**. + +# ARGUMENTS + +Please see *ibv_create_qp_ex(3)* man page for *ibctx* and *ex*. + +## ionic_ex + +```c +enum ionic_qp_init_attr_mask { + IONIC_QP_INIT_ATTR_MASK_FLAGS = 1 << 0, +}; + +enum ionic_qp_init_attr_flags { + IONIC_QP_INIT_ATTR_RCCL = 1 << 16, + IONIC_QP_INIT_ATTR_RCCL_DATA = 1 << 17, + IONIC_QP_INIT_ATTR_RCCL_RDFENCE = 1 << 18, + IONIC_QP_INIT_ATTR_RCCL_RX_OFFLOAD = 1 << 19, + IONIC_QP_INIT_ATTR_RCCL_RCQ = 1 << 24, +}; + +struct ionic_dv_qp_init_attr_ex { + uint64_t comp_mask; + uint32_t flags; +}; +``` + +*comp_mask* +: Bitmask specifying what fields in the structure are valid. + Composed of **ionic_dv_qp_init_attr_mask** values: + + **IONIC_QP_INIT_ATTR_MASK_FLAGS**: + Valid values in *flags*. + +*flags* +: A bitwise OR of the various values described below. Valid when + *comp_mask* includes **IONIC_QP_INIT_ATTR_MASK_FLAGS**. + + **IONIC_QP_INIT_ATTR_RCCL**: + Create an RCCL (RC Collective) QP type. + + **IONIC_QP_INIT_ATTR_RCCL_DATA**: + Enable RCCL data operations on this QP. + + **IONIC_QP_INIT_ATTR_RCCL_RDFENCE**: + Enable RCCL read fence operations on this QP. + + **IONIC_QP_INIT_ATTR_RCCL_RX_OFFLOAD**: + Enable RCCL receive offload operations on this QP. + + **IONIC_QP_INIT_ATTR_RCCL_RCQ**: + Enable Reorder Completion Queue (RCQ) for this QP. When + enabled, out-of-order receive completions are reordered + so that completions are delivered to the application in + order. + +# RETURN VALUE + +**ionic_dv_create_qp_ex()** returns a pointer to the created QP, or NULL +if the request fails (in which case *errno* is set to indicate the error). + +# NOTES + +The *ex->qp_type* field must be **IBV_QPT_RC**. The returned QP always +exposes *qp_type* == **IBV_QPT_RC** to the application. + +Compatibility is handled through the *comp_mask* field. Applications +should only set bits for fields they intend to use. Unknown bits in +*comp_mask* are reserved for future extensions. + +# EXAMPLES + +```c +struct ibv_qp_init_attr_ex qp_attr = { + .send_cq = cq, + .recv_cq = cq, + .cap = { + .max_send_wr = 64, + .max_recv_wr = 64, + .max_send_sge = 4, + .max_recv_sge = 4, + }, + .qp_type = IBV_QPT_RC, + .comp_mask = IBV_QP_INIT_ATTR_PD, + .pd = pd, +}; + +struct ionic_dv_qp_init_attr_ex ionic_attr = { + .comp_mask = IONIC_QP_INIT_ATTR_MASK_FLAGS, + .flags = IONIC_QP_INIT_ATTR_RCCL_RCQ, +}; + +struct ibv_qp *qp = ionic_dv_create_qp_ex(ctx, &qp_attr, &ionic_attr); +``` + +# SEE ALSO + +**ionicdv**(7), +**ibv_create_qp_ex**(3), +**ionic_dv_pd_set_udma_mask**(3), +**ionic_dv_pd_set_sqcmb**(3), +**ionic_dv_pd_set_rqcmb**(3) + +# AUTHORS + +Advanced Micro Devices, Inc.
diff --git a/providers/ionic/man/ionicdv.7.md b/providers/ionic/man/ionicdv.7.md index faf8006..4972f14 100644 --- a/providers/ionic/man/ionicdv.7.md +++ b/providers/ionic/man/ionicdv.7.md
@@ -45,8 +45,19 @@ **ionic_dv_pd_set_rqcmb**(3) to configure CMB preferences on a protection domain before creating queues. +## RCCL and RCQ + +Ionic devices support passing RCCL (RC Collective) flags and Reorder +Completion Queue (RCQ) selection from userspace to firmware at QP creation +time. RCCL flags control data, read fence, and receive offload capabilities +on RC queue pairs. RCQ reorders out-of-order receive completions so that +they are delivered to the application in order. Use +**ionic_dv_create_qp_ex**(3) to create a QP with these ionic-specific +attributes. + # SEE ALSO +**ionic_dv_create_qp_ex**(3), **ionic_dv_ctx_get_udma_count**(3), **ionic_dv_ctx_get_udma_mask**(3), **ionic_dv_pd_get_udma_mask**(3),
diff --git a/providers/irdma/defs.h b/providers/irdma/defs.h index fa522b7..1f4e6d6 100644 --- a/providers/irdma/defs.h +++ b/providers/irdma/defs.h
@@ -114,7 +114,7 @@ #define IRDMAQPSQ_REMSTAGINV GENMASK_ULL(31, 0) #define IRDMAQPSQ_DESTQKEY GENMASK_ULL(31, 0) #define IRDMAQPSQ_DESTQPN GENMASK_ULL(55, 32) -#define IRDMAQPSQ_AHID GENMASK_ULL(16, 0) +#define IRDMAQPSQ_AHID GENMASK_ULL(24, 0) #define IRDMAQPSQ_INLINEDATAFLAG BIT_ULL(57) #define IRDMA_INLINE_VALID_S 7
diff --git a/providers/irdma/osdep.h b/providers/irdma/osdep.h index c9ca5e0..0aa0c99 100644 --- a/providers/irdma/osdep.h +++ b/providers/irdma/osdep.h
@@ -15,10 +15,6 @@ #include <inttypes.h> #include <pthread.h> #include <endian.h> +#include <util/mmio.h> #include <infiniband/verbs.h> - -static inline void db_wr32(__u32 val, __u32 *wqe_word) -{ - *wqe_word = val; -} #endif /* IRDMA_OSDEP_H */
diff --git a/providers/irdma/uk.c b/providers/irdma/uk.c index 85eb735..571ccea 100644 --- a/providers/irdma/uk.c +++ b/providers/irdma/uk.c
@@ -117,7 +117,7 @@ /* valid bit is written before ringing doorbell */ udma_to_device_barrier(); - db_wr32(qp->qp_id, qp->wqe_alloc_db); + mmio_write32(qp->wqe_alloc_db, qp->qp_id); qp->initial_ring.head = qp->sq_ring.head; } @@ -209,6 +209,7 @@ qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id; qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size; qp->sq_wrtrk_array[*wqe_idx].quanta = quanta; + qp->sq_wrtrk_array[*wqe_idx].signaled = info->signaled; return wqe; } @@ -1252,7 +1253,7 @@ udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ - db_wr32(cq->cq_id, cq->cqe_alloc_db); + mmio_write32(cq->cqe_alloc_db, cq->cq_id); } /** @@ -1481,6 +1482,10 @@ info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; if (!info->comp_status) info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len; + if (!qp->sq_wrtrk_array[wqe_idx].signaled) { + ret_code = EFAULT; + goto exit; + } info->op_type = (__u8)FIELD_GET(IRDMACQ_OP, qword3); IRDMA_RING_SET_TAIL(qp->sq_ring, wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta); @@ -1508,6 +1513,10 @@ break; } } while (1); + + if (info->op_type == IRDMA_OP_TYPE_BIND_MW && + info->minor_err == FLUSH_PROT_ERR) + info->minor_err = FLUSH_MW_BIND_ERR; qp->sq_flush_seen = true; if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) qp->sq_flush_complete = true; @@ -1887,6 +1896,9 @@ if (polarity != temp) break; + /* Ensure CQE contents are read after valid bit is checked */ + udma_from_device_barrier(); + get_64bit_val(cqe, 8, &comp_ctx); if ((void *)(uintptr_t)comp_ctx == q) set_64bit_val(cqe, 8, 0);
diff --git a/providers/irdma/user.h b/providers/irdma/user.h index 7460743..9882907 100644 --- a/providers/irdma/user.h +++ b/providers/irdma/user.h
@@ -488,7 +488,8 @@ __u64 wrid; __u32 wr_len; __u16 quanta; - __u8 reserved[2]; + __u8 signaled; + __u8 reserved[1]; }; struct irdma_qp_quanta {
diff --git a/providers/irdma/uverbs.c b/providers/irdma/uverbs.c index 3d1c136..55cf086 100644 --- a/providers/irdma/uverbs.c +++ b/providers/irdma/uverbs.c
@@ -1559,6 +1559,12 @@ info.max_rq_frag_cnt = attr->cap.max_recv_sge; info.max_inline_data = attr->cap.max_inline_data; info.abi_ver = iwvctx->abi_ver; + if (uk_attrs->hw_rev >= IRDMA_GEN_3) { + if (attr->qp_type == IBV_QPT_UD) + info.type = IRDMA_QP_TYPE_ROCE_UD; + else + info.type = IRDMA_QP_TYPE_ROCE_RC; + } status = irdma_uk_calc_depth_shift_sq(&info, &info.sq_depth, &info.sq_shift); if (status) { @@ -1946,7 +1952,10 @@ err = irdma_uk_rdma_read(&iwuqp->qp, &info, false, false); break; case IBV_WR_BIND_MW: - if (ib_qp->qp_type != IBV_QPT_RC) { + if (ib_qp->qp_type != IBV_QPT_RC || + (ib_wr->bind_mw.mw->type == IBV_MW_TYPE_1 && + ib_wr->bind_mw.bind_info.mw_access_flags & + IBV_ACCESS_ZERO_BASED)) { err = EINVAL; break; }
diff --git a/suse/rdma-core.spec b/suse/rdma-core.spec index a9290a2..c3ac21c 100644 --- a/suse/rdma-core.spec +++ b/suse/rdma-core.spec
@@ -388,7 +388,6 @@ %package -n %rdmacm_lname Summary: Userspace RDMA Connection Manager Group: System/Libraries -Requires: %{name} = %{version} Provides: librdmacm = %{version} Obsoletes: librdmacm < %{version}
diff --git a/tests/run_tests.py b/tests/run_tests.py index 9d7edf2..318d152 100755 --- a/tests/run_tests.py +++ b/tests/run_tests.py
@@ -3,12 +3,16 @@ # Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file from args_parser import parser +import importlib.util import unittest +import sys import os -from importlib.machinery import SourceFileLoader module_path = os.path.join(os.path.dirname(__file__), '__init__.py') -tests = SourceFileLoader('tests', module_path).load_module() +spec = importlib.util.spec_from_file_location('tests', module_path) +tests = importlib.util.module_from_spec(spec) +sys.modules['tests'] = tests +spec.loader.exec_module(tests) parser.parse_args() unittest.main(module=tests)
diff --git a/util/mmio.h b/util/mmio.h index df705de..fa12858 100644 --- a/util/mmio.h +++ b/util/mmio.h
@@ -258,7 +258,7 @@ #define mmio_memcpy_x64(dest, src, bytecount) \ ({ \ - if (__builtin_constant_p((bytecount) == 64)) \ + if (__builtin_constant_p(bytecount) && (bytecount) == 64) \ _mmio_memcpy_x64_64b((dest), (src)); \ else \ _mmio_memcpy_x64((dest), (src), (bytecount)); \