Merge pull request #1797 from lishuang1986/lishuang/rc_pingpong-ipv6
rc_pingpong: Prefer IPv6 wildcard when binding control channel
diff --git a/providers/bnxt_re/main.h b/providers/bnxt_re/main.h
index bf8b2de..b699f9b 100644
--- a/providers/bnxt_re/main.h
+++ b/providers/bnxt_re/main.h
@@ -192,6 +192,7 @@
struct bnxt_re_send *cur_sqe;
uint32_t cur_wqe_cnt;
uint32_t cur_slot_cnt;
+ uint32_t used_slot_cnt;
uint32_t cur_swq_idx;
uint8_t cur_opcode;
bool cur_push_wqe;
diff --git a/providers/bnxt_re/memory.h b/providers/bnxt_re/memory.h
index 7412e73..a0089ee 100644
--- a/providers/bnxt_re/memory.h
+++ b/providers/bnxt_re/memory.h
@@ -130,7 +130,7 @@
return que->tail == que->head;
}
-static inline void bnxt_re_incr_tail(struct bnxt_re_queue *que, uint8_t cnt)
+static inline void bnxt_re_incr_tail(struct bnxt_re_queue *que, uint32_t cnt)
{
que->tail += cnt;
if (que->tail >= que->depth) {
diff --git a/providers/bnxt_re/verbs.c b/providers/bnxt_re/verbs.c
index 5fe67a8..a620c0b 100644
--- a/providers/bnxt_re/verbs.c
+++ b/providers/bnxt_re/verbs.c
@@ -1614,7 +1614,7 @@
if (send_flags & IBV_SEND_INLINE)
hdrval |= ((BNXT_RE_WR_FLAGS_INLINE & BNXT_RE_HDR_FLAGS_MASK)
<< BNXT_RE_HDR_FLAGS_SHIFT);
- hdrval |= ((qp->wr_sq.cur_slot_cnt) & BNXT_RE_HDR_WS_MASK) << BNXT_RE_HDR_WS_SHIFT;
+ hdrval |= ((qp->wr_sq.used_slot_cnt) & BNXT_RE_HDR_WS_MASK) << BNXT_RE_HDR_WS_SHIFT;
opcd = bnxt_re_ibv_to_bnxt_wr_opcd(qp->wr_sq.cur_opcode);
hdrval |= (opcd & BNXT_RE_HDR_WT_MASK);
qp->wr_sq.cur_hdr->rsv_ws_fl_wt = htole32(hdrval);
@@ -1693,7 +1693,7 @@
wrid->wrid = ibvqp->wr_id;
wrid->bytes = length;
wrid->slots = (qp->qpmode == BNXT_RE_WQE_MODE_STATIC) ?
- STATIC_WQE_NUM_SLOTS : qp->wr_sq.cur_slot_cnt;
+ STATIC_WQE_NUM_SLOTS : qp->wr_sq.used_slot_cnt;
wrid->sig = (ibvqp->wr_flags & IBV_SEND_SIGNALED || qp->cap.sqsig) ?
IBV_SEND_SIGNALED : 0;
wrid->wc_opcd = bnxt_re_ibv_wr_to_wc_opcd(qp->wr_sq.cur_opcode);
@@ -1708,6 +1708,7 @@
qp->wr_sq.cur_hdr = NULL;
qp->wr_sq.cur_sqe = NULL;
qp->wr_sq.cur_slot_cnt = 0;
+ qp->wr_sq.used_slot_cnt = 0;
qp->wr_sq.cur_wqe_cnt = 0;
qp->wr_sq.cur_opcode = 0xff;
qp->wr_sq.cur_push_wqe = false;
@@ -1720,11 +1721,10 @@
struct bnxt_re_qp *qp = to_bnxt_re_qp((struct ibv_qp *)ibvqp);
struct bnxt_re_queue *sq = qp->jsqq->hwque;
int err = qp->wr_sq.error;
- uint8_t slots;
+ uint32_t slots;
if (unlikely(err))
goto exit;
- bnxt_re_set_wr_hdr_flags(qp, ibvqp->wr_flags);
qp->wqe_cnt += qp->wr_sq.cur_wqe_cnt;
slots = (qp->qpmode == BNXT_RE_WQE_MODE_STATIC) ?
STATIC_WQE_NUM_SLOTS : qp->wr_sq.cur_slot_cnt;
@@ -1775,8 +1775,10 @@
else
bnxt_re_fill_psns(qp, length, *sq->dbtail, qp->wr_sq.cur_opcode);
+ qp->wr_sq.used_slot_cnt = 3;
bnxt_re_update_swqe(ibvqp, qp, length);
qp->wr_sq.cur_wqe_cnt++;
+ bnxt_re_set_wr_hdr_flags(qp, ibvqp->wr_flags);
}
static void bnxt_re_send_wr_set_sge_list(struct ibv_qp_ex *ibvqp, size_t nsge,
@@ -1816,8 +1818,10 @@
else
bnxt_re_fill_psns(qp, len, *sq->dbtail, qp->wr_sq.cur_opcode);
+ qp->wr_sq.used_slot_cnt = nsge + 2;
bnxt_re_update_swqe(ibvqp, qp, len);
qp->wr_sq.cur_wqe_cnt++;
+ bnxt_re_set_wr_hdr_flags(qp, ibvqp->wr_flags);
}
static void bnxt_re_send_wr_set_inline_data(struct ibv_qp_ex *ibvqp,
@@ -1827,6 +1831,7 @@
struct bnxt_re_queue *sq = qp->jsqq->hwque;
struct bnxt_re_push_buffer *pushb = NULL;
struct ibv_data_buf ibv_buf;
+ uint32_t wrd_slot_cnt;
uint32_t len = 0;
if (unlikely(qp->wr_sq.error))
@@ -1839,6 +1844,7 @@
}
ibv_buf.addr = addr;
ibv_buf.length = length;
+ wrd_slot_cnt = (length + MSG_LEN_ADJ_TO_BYTES) >> SLOTS_RSH_TO_NUM_WQE;
len = bnxt_re_put_wr_inline(sq, &qp->wr_sq.cur_slot_cnt, pushb, 1, &ibv_buf, &length);
if (qp->qptyp == IBV_QPT_UD) {
qp->wr_sq.cur_hdr->lhdr.qkey_len |= htole64(len);
@@ -1851,9 +1857,11 @@
bnxt_re_fill_psns_for_msntbl(qp, len, *sq->dbtail, qp->wr_sq.cur_wqe_cnt);
else
bnxt_re_fill_psns(qp, len, *sq->dbtail, qp->wr_sq.cur_opcode);
+ qp->wr_sq.used_slot_cnt = wrd_slot_cnt + 2;
bnxt_re_update_swqe(ibvqp, qp, len);
qp->wr_sq.cur_wqe_cnt++;
qp->wr_sq.cur_push_size += length;
+ bnxt_re_set_wr_hdr_flags(qp, ibvqp->wr_flags);
}
static void bnxt_re_send_wr_set_inline_data_list(struct ibv_qp_ex *ibvqp, size_t num_buf,
@@ -1894,9 +1902,11 @@
bnxt_re_fill_psns_for_msntbl(qp, len, *sq->dbtail, qp->wr_sq.cur_opcode);
else
bnxt_re_fill_psns(qp, len, *sq->dbtail, qp->wr_sq.cur_opcode);
+ qp->wr_sq.used_slot_cnt = num + 2;
bnxt_re_update_swqe(ibvqp, qp, len);
qp->wr_sq.cur_wqe_cnt++;
qp->wr_sq.cur_push_size += msg_len;
+ bnxt_re_set_wr_hdr_flags(qp, ibvqp->wr_flags);
}
static void bnxt_re_send_wr_set_ud_addr(struct ibv_qp_ex *ibvqp, struct ibv_ah *ibah,
diff --git a/pyverbs/providers/mlx5/mlx5dv.pyx b/pyverbs/providers/mlx5/mlx5dv.pyx
index 00dd977..7f726ec 100644
--- a/pyverbs/providers/mlx5/mlx5dv.pyx
+++ b/pyverbs/providers/mlx5/mlx5dv.pyx
@@ -1998,6 +1998,12 @@
if self.addr:
return <uintptr_t><void*>self.addr
+ @umem_addr.setter
+ def umem_addr(self, addr):
+ if not self.is_user_addr and self.addr != NULL:
+ raise PyverbsUserError('Cannot override the address of an internally allocated UMEM')
+ self.addr = <void*><uintptr_t>addr
+ self.is_user_addr = True
cdef class Mlx5Cqe64(PyverbsObject):
def __init__(self, addr):
diff --git a/tests/test_mlx5_devx.py b/tests/test_mlx5_devx.py
index 7999a9f..cc0e420 100644
--- a/tests/test_mlx5_devx.py
+++ b/tests/test_mlx5_devx.py
@@ -8,8 +8,11 @@
import resource
import unittest
import errno
+import os
from tests.mlx5_base import Mlx5DevxRcResources, Mlx5DevxTrafficBase
+from tests.test_buf import alloc_buf, device_has_cc_dma_bounce, \
+ make_cc_pd, register_buf_mr
from pyverbs.providers.mlx5.mlx5dv import Mlx5Context, Mlx5DVContextAttr, \
Mlx5DevxCmdComp, Mlx5DevxObj, Mlx5UMEM
from pyverbs.providers.mlx5.mlx5_enums import mlx5dv_context_attr_flags
@@ -20,6 +23,72 @@
import tests.utils as u
+class BufDevxRcResources(Mlx5DevxRcResources):
+ """
+ DevX RC resources for a Confidential Computing (CoCo) guest: every DevX
+ UMEM and the data MR live in shared/unprotected memory allocated with
+ Buf on a CC parent domain. Each UMEM is registered through the
+ DevX dmabuf path using an FD exported from its Buf, so all NIC-DMA'd memory
+ is shared as a DMA-bounce device requires.
+ """
+ def __init__(self, dev_name, ib_port, gid_index, msg_size=1024,
+ activate_port_state=False, send_dbr_mode=0):
+ self.bufs = []
+ self.dmabuf_fds = []
+ self.base_pd = None
+ super().__init__(dev_name, ib_port, gid_index, msg_size,
+ activate_port_state, send_dbr_mode)
+
+ def create_pd(self):
+ """Build a CC parent domain and derive the DevX pdn from it."""
+ from pyverbs.providers.mlx5.mlx5dv_objects import Mlx5DvObj
+ from pyverbs.providers.mlx5.mlx5_enums import mlx5dv_obj_type
+ if not device_has_cc_dma_bounce(self.ctx):
+ raise unittest.SkipTest('Device is not a CC DMA-bounce device')
+ self.base_pd, self.pd = make_cc_pd(self.ctx)
+ self.dv_pd = Mlx5DvObj(mlx5dv_obj_type.MLX5DV_OBJ_PD, pd=self.pd).dvpd
+
+ def create_mr(self):
+ """Register the data buffer as a shared Buf MR on the CC PD."""
+ access = ibv_access_flags.IBV_ACCESS_REMOTE_WRITE | \
+ ibv_access_flags.IBV_ACCESS_LOCAL_WRITE | \
+ ibv_access_flags.IBV_ACCESS_REMOTE_READ
+ buf = alloc_buf(self.pd, self.msg_size)
+ self.mr = register_buf_mr(self.pd, buf, self.msg_size, access)
+
+ def create_umem(self, size, access=ibv_access_flags.IBV_ACCESS_LOCAL_WRITE,
+ alignment=resource.getpagesize()):
+ """Return a DevX UMEM backed by a Buf exported as a dmabuf FD."""
+ from pyverbs.providers.mlx5.mlx5_enums import MLX5DV_UMEM_MASK_DMABUF
+ page_size = resource.getpagesize()
+ alloc_size = max(size, page_size)
+ buf = alloc_buf(self.pd, alloc_size)
+ mem.write(buf.addr, bytes(alloc_size), alloc_size) # Zero-fill the buffer
+ fd = self.export_buf_dmabuf_fd(buf)
+ umem = Mlx5UMEM(self.ctx, alloc_size, addr=0, alignment=alignment, access=access,
+ pgsz_bitmap=page_size, comp_mask=MLX5DV_UMEM_MASK_DMABUF, dmabuf_fd=fd)
+ umem.umem_addr = buf.addr
+ self.bufs.append(buf)
+ self.dmabuf_fds.append(fd)
+ return umem
+
+ def export_buf_dmabuf_fd(self, buf):
+ """Export the Buf's dmabuf FD, skipping when it is not dmabuf-backed."""
+ try:
+ return buf.export_dmabuf_fd()
+ except PyverbsRDMAError as ex:
+ if ex.error_code == errno.ENODATA:
+ raise unittest.SkipTest('Buf is not dmabuf-backed')
+ raise
+
+ def close_resources(self):
+ """Close the exported dmabuf FDs"""
+ super().close_resources()
+ for fd in self.dmabuf_fds:
+ os.close(fd)
+ self.dmabuf_fds = []
+
+
class Mlx5DevxRcOdpRes(Mlx5DevxRcResources):
@u.requires_odpv2
def create_mr(self):
@@ -45,6 +114,15 @@
# Send traffic
self.send_imm_traffic()
+ def test_devx_rc_qp_send_imm_buf_umem_traffic(self):
+ """
+ Run DevX RC SEND_IMM traffic where all NIC memory (QP, CQ, doorbell
+ UMEMs and the data MR) is shared CoCo memory: buffers on a CC parent
+ domain, registered as UMEMs via their dmabuf FD.
+ """
+ self.create_players(BufDevxRcResources)
+ self.send_imm_traffic()
+
def test_devx_rc_qp_send_imm_doorbell_less_traffic(self):
"""
Creates two DevX RC QPs with dbr less ext and modifies them to RTS state.
diff --git a/util/udma_barrier.h b/util/udma_barrier.h
index c0cd5f0..bfad001 100644
--- a/util/udma_barrier.h
+++ b/util/udma_barrier.h
@@ -102,6 +102,9 @@
#define udma_to_device_barrier() asm volatile("fence ow,ow" ::: "memory")
#elif defined(__mips__)
#define udma_to_device_barrier() asm volatile("sync" ::: "memory")
+#elif defined(__e2k__)
+#include <e2kbuiltin.h>
+#define udma_to_device_barrier() __builtin_ia32_mfence()
#else
#error No architecture specific memory barrier defines found!
#endif
@@ -140,6 +143,8 @@
#define udma_from_device_barrier() asm volatile("fence ir,ir" ::: "memory")
#elif defined(__mips__)
#define udma_from_device_barrier() asm volatile("sync" ::: "memory")
+#elif defined(__e2k__)
+#define udma_from_device_barrier() __builtin_ia32_lfence()
#else
#error No architecture specific memory barrier defines found!
#endif
@@ -213,6 +218,8 @@
#define mmio_flush_writes() s390_pciwb()
#elif defined(__mips__)
#define mmio_flush_writes() asm volatile("sync" ::: "memory")
+#elif defined(__e2k__)
+#define mmio_flush_writes() __builtin_ia32_sfence()
#else
#error No architecture specific memory barrier defines found!
#endif
