efa: Add Completion Counters support Implement completion counters for the EFA provider. Each counter object has two event counters (success and error) with set, inc, and read operations. Counter values are accessed directly from userspace through mapped memory. Add efadv_create_comp_cntr() for extended creation with external memory options. Buffer memory is described to the kernel via ib_uverbs_buffer_desc struct passed as a driver-specific ioctl attribute. Three modes are supported per counter: user-supplied VA, user-supplied dmabuf, or provider-internal memory. Reviewed-by: Yonatan Nachum <ynachum@amazon.com> Signed-off-by: Michael Margolin <mrgolin@amazon.com>
diff --git a/debian/ibverbs-providers.symbols b/debian/ibverbs-providers.symbols index 3697696..40e627d 100644 --- a/debian/ibverbs-providers.symbols +++ b/debian/ibverbs-providers.symbols
@@ -188,6 +188,7 @@ EFA_1.4@EFA_1.4 59 EFA_1.5@EFA_1.5 63 EFA_1.6@EFA_1.6 64 + EFA_1.7@EFA_1.7 65 efadv_create_driver_qp@EFA_1.0 24 efadv_create_qp_ex@EFA_1.1 26 efadv_query_device@EFA_1.1 26 @@ -200,6 +201,7 @@ efadv_get_max_sq_depth@EFA_1.5 63 efadv_get_max_rq_depth@EFA_1.5 63 efadv_qp_from_ibv_qp_ex@EFA_1.6 64 + efadv_create_comp_cntr@EFA_1.7 65 libhns.so.1 ibverbs-providers #MINVER# * Build-Depends-Package: libibverbs-dev HNS_1.0@HNS_1.0 51
diff --git a/providers/efa/CMakeLists.txt b/providers/efa/CMakeLists.txt index 1f3c32c..4054109 100644 --- a/providers/efa/CMakeLists.txt +++ b/providers/efa/CMakeLists.txt
@@ -3,7 +3,7 @@ endif() rdma_shared_provider(efa libefa.map - 1 1.6.${PACKAGE_VERSION} + 1 1.7.${PACKAGE_VERSION} ${TRACE_FILE} efa.c verbs.c
diff --git a/providers/efa/efa.c b/providers/efa/efa.c index 5e6d873..7ef6ff3 100644 --- a/providers/efa/efa.c +++ b/providers/efa/efa.c
@@ -31,30 +31,40 @@ .alloc_pd = efa_alloc_pd, .alloc_parent_domain = efa_alloc_parent_domain, .alloc_td = efa_alloc_td, + .cq_event = efa_cq_event, .create_ah = efa_create_ah, + .create_comp_cntr = efa_create_comp_cntr, .create_cq = efa_create_cq, .create_cq_ex = efa_create_cq_ex, .create_qp = efa_create_qp, .create_qp_ex = efa_create_qp_ex, - .cq_event = efa_cq_event, .dealloc_pd = efa_dealloc_pd, .dealloc_td = efa_dealloc_td, .dereg_mr = efa_dereg_mr, .destroy_ah = efa_destroy_ah, + .destroy_comp_cntr = efa_destroy_comp_cntr, .destroy_cq = efa_destroy_cq, .destroy_qp = efa_destroy_qp, + .inc_comp_cntr = efa_inc_comp_cntr, + .inc_err_comp_cntr = efa_inc_err_comp_cntr, .modify_qp = efa_modify_qp, .poll_cq = efa_poll_cq, .post_recv = efa_post_recv, .post_send = efa_post_send, + .qp_attach_comp_cntr = efa_qp_attach_comp_cntr, + .query_comp_cntr_caps = ibv_cmd_query_comp_cntr_caps, .query_device_ex = efa_query_device_ex, .query_port = efa_query_port, .query_port_speed = efa_query_port_speed, .query_qp = efa_query_qp, .query_qp_data_in_order = efa_query_qp_data_in_order, + .read_comp_cntr = efa_read_comp_cntr, + .read_err_comp_cntr = efa_read_err_comp_cntr, .reg_dmabuf_mr = efa_reg_dmabuf_mr, .reg_mr = efa_reg_mr, .req_notify_cq = efa_arm_cq, + .set_comp_cntr = efa_set_comp_cntr, + .set_err_comp_cntr = efa_set_err_comp_cntr, .free_context = efa_free_context, };
diff --git a/providers/efa/efa.h b/providers/efa/efa.h index 7a596c9..20567cc 100644 --- a/providers/efa/efa.h +++ b/providers/efa/efa.h
@@ -105,6 +105,14 @@ struct efa_sub_cq sub_cq_arr[]; }; +struct efa_comp_cntr { + struct ibv_comp_cntr ibv_comp_cntr; + uint64_t comp_val __attribute__((aligned(8))); + uint64_t err_val __attribute__((aligned(8))); + uint64_t *comp_ptr; + uint64_t *err_ptr; +}; + struct efa_wq { uint64_t *wrid; /* wrid_idx_pool: Pool of free indexes in the wrid array, used to select the @@ -230,6 +238,11 @@ return container_of(efadv_cq, struct efa_cq, dv_cq); } +static inline struct efa_comp_cntr *to_efa_comp_cntr(struct ibv_comp_cntr *ibvcc) +{ + return container_of(ibvcc, struct efa_comp_cntr, ibv_comp_cntr); +} + static inline struct efa_qp *to_efa_qp(struct ibv_qp *ibvqp) { return container_of(ibvqp, struct efa_qp, verbs_qp.qp);
diff --git a/providers/efa/efadv.h b/providers/efa/efadv.h index 8745988..d863de4 100644 --- a/providers/efa/efadv.h +++ b/providers/efa/efadv.h
@@ -23,6 +23,7 @@ EFADV_DEVICE_ATTR_CAPS_RDMA_WRITE = 1 << 3, EFADV_DEVICE_ATTR_CAPS_UNSOLICITED_WRITE_RECV = 1 << 4, EFADV_DEVICE_ATTR_CAPS_CQ_WITH_EXT_MEM_DMABUF = 1 << 5, + EFADV_DEVICE_ATTR_CAPS_COMP_CNTR = 1 << 6, }; struct efadv_device_attr { @@ -191,6 +192,40 @@ } enum { + EFADV_MEMORY_LOCATION_VA, + EFADV_MEMORY_LOCATION_DMABUF, +}; + +struct efadv_memory_location { + uint8_t *ptr; + struct { + uint64_t offset; + int32_t fd; + uint32_t reserved; + } dmabuf; + uint8_t type; + uint8_t reserved[7]; +}; + +enum { + EFADV_COMP_CNTR_INIT_WITH_COMP_EXTERNAL_MEM = 1 << 0, + EFADV_COMP_CNTR_INIT_WITH_ERR_EXTERNAL_MEM = 1 << 1, +}; + +struct efadv_comp_cntr_init_attr { + uint64_t comp_mask; + uint32_t flags; + uint32_t reserved; + struct efadv_memory_location comp_cntr_ext_mem; + struct efadv_memory_location err_cntr_ext_mem; +}; + +struct ibv_comp_cntr *efadv_create_comp_cntr(struct ibv_context *ibvctx, + struct ibv_comp_cntr_init_attr *attr, + struct efadv_comp_cntr_init_attr *efa_attr, + uint32_t inlen); + +enum { EFADV_MR_ATTR_VALIDITY_RECV_IC_ID = 1 << 0, EFADV_MR_ATTR_VALIDITY_RDMA_READ_IC_ID = 1 << 1, EFADV_MR_ATTR_VALIDITY_RDMA_RECV_IC_ID = 1 << 2,
diff --git a/providers/efa/libefa.map b/providers/efa/libefa.map index 3f51e5e..dee028a 100644 --- a/providers/efa/libefa.map +++ b/providers/efa/libefa.map
@@ -40,3 +40,8 @@ global: efadv_qp_from_ibv_qp_ex; } EFA_1.5; + +EFA_1.7 { + global: + efadv_create_comp_cntr; +} EFA_1.6;
diff --git a/providers/efa/man/efadv_create_comp_cntr.3.md b/providers/efa/man/efadv_create_comp_cntr.3.md new file mode 100644 index 0000000..92d92dc --- /dev/null +++ b/providers/efa/man/efadv_create_comp_cntr.3.md
@@ -0,0 +1,118 @@ +--- +layout: page +title: EFADV_CREATE_COMP_CNTR +section: 3 +tagline: Verbs +date: 2026-04-27 +header: "EFA Direct Verbs Manual" +footer: efa +--- + +# NAME + +efadv_create_comp_cntr - Create EFA specific Completion Counter + +# SYNOPSIS + +```c +#include <infiniband/efadv.h> + +struct ibv_comp_cntr *efadv_create_comp_cntr(struct ibv_context *context, + struct ibv_comp_cntr_init_attr *attr, + struct efadv_comp_cntr_init_attr *efa_attr, + uint32_t inlen); +``` + +# DESCRIPTION + +**efadv_create_comp_cntr()** creates a Completion Counter with EFA specific +properties, such as external memory for the counter values. + +The argument *attr* is an ibv_comp_cntr_init_attr struct, as defined in +<infiniband/verbs.h>. + +Compatibility is handled using the comp_mask and inlen fields. + +```c +enum { + EFADV_MEMORY_LOCATION_VA, + EFADV_MEMORY_LOCATION_DMABUF, +}; + +struct efadv_memory_location { + uint8_t *ptr; + struct { + uint64_t offset; + int32_t fd; + uint32_t reserved; + } dmabuf; + uint8_t type; + uint8_t reserved[7]; +}; + +struct efadv_comp_cntr_init_attr { + uint64_t comp_mask; + uint32_t flags; + uint32_t reserved; + struct efadv_memory_location comp_cntr_ext_mem; + struct efadv_memory_location err_cntr_ext_mem; +}; +``` + +*inlen* +: In: Size of struct efadv_comp_cntr_init_attr. + +*comp_mask* +: Compatibility mask. + +*flags* +: A bitwise OR of the various values described below. + + **EFADV_COMP_CNTR_INIT_WITH_COMP_EXTERNAL_MEM**: + Use application-provided memory for the completion count, as + described by *comp_cntr_ext_mem*. + + **EFADV_COMP_CNTR_INIT_WITH_ERR_EXTERNAL_MEM**: + Use application-provided memory for the error count, as + described by *err_cntr_ext_mem*. + +*comp_cntr_ext_mem* +: Memory location for the completion count when using external memory. + +*err_cntr_ext_mem* +: Memory location for the error count when using external memory. + +## efadv_memory_location + +The external memory is described by an **efadv_memory_location** structure +which supports two modes: + +*type* +: **EFADV_MEMORY_LOCATION_VA** for a virtual address, or + **EFADV_MEMORY_LOCATION_DMABUF** for a DMA-BUF reference. + +*ptr* +: Virtual address pointer. Required when type is + **EFADV_MEMORY_LOCATION_VA**. When type is + **EFADV_MEMORY_LOCATION_DMABUF**, may optionally be set to provide a + process-accessible mapping of the DMA-BUF memory. + +*dmabuf.fd* +: DMA-BUF file descriptor (used when type is + **EFADV_MEMORY_LOCATION_DMABUF**). + +*dmabuf.offset* +: Offset within the DMA-BUF. + +# RETURN VALUE + +efadv_create_comp_cntr() returns a pointer to the created ibv_comp_cntr, or +NULL if the request fails. + +# SEE ALSO + +**efadv**(7), **ibv_create_comp_cntr**(3), **ibv_qp_attach_comp_cntr**(3) + +# AUTHORS + +Michael Margolin <mrgolin@amazon.com>
diff --git a/providers/efa/man/efadv_query_device.3.md b/providers/efa/man/efadv_query_device.3.md index f46d362..cff7068 100644 --- a/providers/efa/man/efadv_query_device.3.md +++ b/providers/efa/man/efadv_query_device.3.md
@@ -92,6 +92,9 @@ Indicates that creating CQs with external memory buffers by passing dmabuf is supported. + EFADV_DEVICE_ATTR_CAPS_COMP_CNTR: + Completion counters are supported. + *max_rdma_size* : Maximum RDMA transfer size in bytes.
diff --git a/providers/efa/verbs.c b/providers/efa/verbs.c index 8327cb3..aa76976 100644 --- a/providers/efa/verbs.c +++ b/providers/efa/verbs.c
@@ -198,6 +198,9 @@ if (EFA_DEV_CAP(ctx, CQ_WITH_EXT_MEM)) attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_CQ_WITH_EXT_MEM_DMABUF; + + if (EFA_DEV_CAP(ctx, COMP_CNTR)) + attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_COMP_CNTR; } if (vext_field_avail(typeof(*attr), max_rdma_size, inlen)) { @@ -1404,6 +1407,183 @@ return 0; } +static void efa_fill_buffer_desc_va(struct ib_uverbs_buffer_desc *desc, uint64_t addr, + uint64_t length) +{ + desc->type = IB_UVERBS_BUFFER_TYPE_VA; + desc->addr = addr; + desc->length = length; +} + +static void efa_fill_buffer_desc_dmabuf(struct ib_uverbs_buffer_desc *desc, int32_t fd, + uint64_t offset, uint64_t length) +{ + desc->type = IB_UVERBS_BUFFER_TYPE_DMABUF; + desc->fd = fd; + desc->addr = offset; + desc->length = length; +} + +static void efa_fill_buffer_desc_from_mem_loc(struct ib_uverbs_buffer_desc *desc, + struct efadv_memory_location *mem, uint64_t length) +{ + if (mem->type == EFADV_MEMORY_LOCATION_DMABUF) + efa_fill_buffer_desc_dmabuf(desc, mem->dmabuf.fd, mem->dmabuf.offset, length); + else + efa_fill_buffer_desc_va(desc, (uintptr_t)mem->ptr, length); +} + +static inline bool efa_comp_cntr_mem_type_supported(uint32_t mem_type) +{ + return mem_type == EFADV_MEMORY_LOCATION_VA || mem_type == EFADV_MEMORY_LOCATION_DMABUF; +} + +static struct ibv_comp_cntr *efa_create_comp_cntr_impl(struct ibv_context *ibvctx, + struct ibv_comp_cntr_init_attr *attr, + struct efadv_comp_cntr_init_attr *efa_attr) +{ + uint32_t supported_efa_flags = EFADV_COMP_CNTR_INIT_WITH_COMP_EXTERNAL_MEM | + EFADV_COMP_CNTR_INIT_WITH_ERR_EXTERNAL_MEM; + DECLARE_COMMAND_BUFFER_LINK(cmdb, UVERBS_OBJECT_COMP_CNTR, + UVERBS_METHOD_COMP_CNTR_CREATE, 2, NULL); + struct ib_uverbs_buffer_desc comp_desc = {}; + struct ib_uverbs_buffer_desc err_desc = {}; + struct efa_comp_cntr *cc; + int err; + + if (attr->comp_mask || attr->flags || + attr->type != IBV_COMP_CNTR_TYPE_WRS || + efa_attr->comp_mask || + !check_comp_mask(efa_attr->flags, supported_efa_flags) || + !efa_comp_cntr_mem_type_supported(efa_attr->comp_cntr_ext_mem.type) || + !efa_comp_cntr_mem_type_supported(efa_attr->err_cntr_ext_mem.type)) { + verbs_err(verbs_get_ctx(ibvctx), "Unsupported type or flag\n"); + errno = EOPNOTSUPP; + return NULL; + } + + cc = calloc(1, sizeof(*cc)); + if (!cc) { + errno = ENOMEM; + return NULL; + } + + if (efa_attr->flags & EFADV_COMP_CNTR_INIT_WITH_COMP_EXTERNAL_MEM) { + efa_fill_buffer_desc_from_mem_loc(&comp_desc, &efa_attr->comp_cntr_ext_mem, + sizeof(uint64_t)); + cc->comp_ptr = (uint64_t *)efa_attr->comp_cntr_ext_mem.ptr; + } else { + efa_fill_buffer_desc_va(&comp_desc, (uintptr_t)&cc->comp_val, sizeof(uint64_t)); + cc->comp_ptr = &cc->comp_val; + } + + fill_attr_in_ptr(cmdb, EFA_IB_ATTR_CREATE_COMP_CNTR_COMP_BUFFER, &comp_desc); + + if (efa_attr->flags & EFADV_COMP_CNTR_INIT_WITH_ERR_EXTERNAL_MEM) { + efa_fill_buffer_desc_from_mem_loc(&err_desc, &efa_attr->err_cntr_ext_mem, + sizeof(uint64_t)); + cc->err_ptr = (uint64_t *)efa_attr->err_cntr_ext_mem.ptr; + } else { + efa_fill_buffer_desc_va(&err_desc, (uintptr_t)&cc->err_val, sizeof(uint64_t)); + cc->err_ptr = &cc->err_val; + } + + fill_attr_in_ptr(cmdb, EFA_IB_ATTR_CREATE_COMP_CNTR_ERR_BUFFER, &err_desc); + + err = ibv_cmd_create_comp_cntr(ibvctx, &cc->ibv_comp_cntr, cmdb); + if (err) { + free(cc); + errno = err; + return NULL; + } + + return &cc->ibv_comp_cntr; +} + +struct ibv_comp_cntr *efa_create_comp_cntr(struct ibv_context *ibvctx, + struct ibv_comp_cntr_init_attr *attr) +{ + struct efadv_comp_cntr_init_attr efa_attr = {}; + + return efa_create_comp_cntr_impl(ibvctx, attr, &efa_attr); +} + +struct ibv_comp_cntr *efadv_create_comp_cntr(struct ibv_context *ibvctx, + struct ibv_comp_cntr_init_attr *attr, + struct efadv_comp_cntr_init_attr *efa_attr, + uint32_t inlen) +{ + if (!is_efa_dev(ibvctx->device)) { + verbs_err(verbs_get_ctx(ibvctx), "Not an EFA device\n"); + errno = EOPNOTSUPP; + return NULL; + } + + if (!vext_field_avail(struct efadv_comp_cntr_init_attr, err_cntr_ext_mem, inlen) || + (inlen > sizeof(*efa_attr) && !is_ext_cleared(efa_attr, inlen))) { + verbs_err(verbs_get_ctx(ibvctx), "Compatibility issues\n"); + errno = EINVAL; + return NULL; + } + + return efa_create_comp_cntr_impl(ibvctx, attr, efa_attr); +} + +int efa_destroy_comp_cntr(struct ibv_comp_cntr *ibvcc) +{ + struct efa_comp_cntr *cc = to_efa_comp_cntr(ibvcc); + int err; + + err = ibv_cmd_destroy_comp_cntr(ibvcc); + if (err) + return err; + + free(cc); + return 0; +} + +int efa_set_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t value) +{ + return ibv_cmd_set_comp_cntr(ibvcc, value); +} + +int efa_set_err_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t value) +{ + return ibv_cmd_set_err_comp_cntr(ibvcc, value); +} + +int efa_inc_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t amount) +{ + return ibv_cmd_inc_comp_cntr(ibvcc, amount); +} + +int efa_inc_err_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t amount) +{ + return ibv_cmd_inc_err_comp_cntr(ibvcc, amount); +} + +int efa_read_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t *value) +{ + struct efa_comp_cntr *cc = to_efa_comp_cntr(ibvcc); + + if (!cc->comp_ptr) + return EOPNOTSUPP; + + *value = *cc->comp_ptr; + return 0; +} + +int efa_read_err_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t *value) +{ + struct efa_comp_cntr *cc = to_efa_comp_cntr(ibvcc); + + if (!cc->err_ptr) + return EOPNOTSUPP; + + *value = *cc->err_ptr; + return 0; +} + static void efa_wq_terminate(struct efa_wq *wq, int pgsz) { void *db_aligned; @@ -2145,6 +2325,12 @@ return 0; } +int efa_qp_attach_comp_cntr(struct ibv_qp *qp, struct ibv_comp_cntr *comp_cntr, + struct ibv_qp_attach_comp_cntr_attr *attr) +{ + return ibv_cmd_qp_attach_comp_cntr(qp, comp_cntr, attr); +} + int efa_query_qp(struct ibv_qp *ibvqp, struct ibv_qp_attr *attr, int attr_mask, struct ibv_qp_init_attr *init_attr) {
diff --git a/providers/efa/verbs.h b/providers/efa/verbs.h index b38b4e4..602a8d1 100644 --- a/providers/efa/verbs.h +++ b/providers/efa/verbs.h
@@ -36,12 +36,24 @@ int efa_arm_cq(struct ibv_cq *ibvcq, int solicited_only); void efa_cq_event(struct ibv_cq *ibvcq); +struct ibv_comp_cntr *efa_create_comp_cntr(struct ibv_context *ibvctx, + struct ibv_comp_cntr_init_attr *attr); +int efa_destroy_comp_cntr(struct ibv_comp_cntr *ibvcc); +int efa_set_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t value); +int efa_set_err_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t value); +int efa_inc_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t amount); +int efa_inc_err_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t amount); +int efa_read_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t *value); +int efa_read_err_comp_cntr(struct ibv_comp_cntr *ibvcc, uint64_t *value); + struct ibv_qp *efa_create_qp(struct ibv_pd *ibvpd, struct ibv_qp_init_attr *attr); struct ibv_qp *efa_create_qp_ex(struct ibv_context *ibvctx, struct ibv_qp_init_attr_ex *attr_ex); int efa_modify_qp(struct ibv_qp *ibvqp, struct ibv_qp_attr *attr, int ibv_qp_attr_mask); +int efa_qp_attach_comp_cntr(struct ibv_qp *qp, struct ibv_comp_cntr *comp_cntr, + struct ibv_qp_attach_comp_cntr_attr *attr); int efa_query_qp(struct ibv_qp *ibvqp, struct ibv_qp_attr *attr, int attr_mask, struct ibv_qp_init_attr *init_attr); int efa_query_qp_data_in_order(struct ibv_qp *ibvqp, enum ibv_wr_opcode op,