Merge pull request #1790 from selvintxavier/wr_api_fixes
bnxt_re/lib: Fixes in WR APIs
diff --git a/buildlib/pyverbs_functions.cmake b/buildlib/pyverbs_functions.cmake
index 57aec04..868b75e 100644
--- a/buildlib/pyverbs_functions.cmake
+++ b/buildlib/pyverbs_functions.cmake
@@ -61,6 +61,13 @@
endfunction()
function(rdma_python_module PY_MODULE)
+ if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/__version__.py.in")
+ message(FATAL_ERROR "Can't generate python module ${PY_MODULE} version, need ${CMAKE_CURRENT_SOURCE_DIR}/__version__.py.in")
+ endif()
+ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/__version__.py.in"
+ "${BUILD_PYTHON}/${PY_MODULE}/__version__.py" @ONLY)
+ install(FILES "${BUILD_PYTHON}/${PY_MODULE}/__version__.py"
+ DESTINATION ${CMAKE_INSTALL_PYTHON_ARCH_LIB}/${PY_MODULE})
foreach(PY_FILE ${ARGN})
get_filename_component(LINK "${CMAKE_CURRENT_SOURCE_DIR}/${PY_FILE}" ABSOLUTE)
rdma_create_symlink("${LINK}" "${BUILD_PYTHON}/${PY_MODULE}/${PY_FILE}")
diff --git a/debian/ibverbs-providers.symbols b/debian/ibverbs-providers.symbols
index 40e627d..044b4c9 100644
--- a/debian/ibverbs-providers.symbols
+++ b/debian/ibverbs-providers.symbols
@@ -211,8 +211,10 @@
libmana.so.1 ibverbs-providers #MINVER#
* Build-Depends-Package: libibverbs-dev
MANA_1.0@MANA_1.0 41
+ MANA_1.1@MANA_1.1 65
manadv_init_obj@MANA_1.0 41
manadv_set_context_attr@MANA_1.0 41
+ manadv_alloc_pd@MANA_1.1 65
libionic.so.1 ibverbs-providers #MINVER#
* Build-Depends-Package: libibverbs-dev
IONIC_1.0@IONIC_1.0 59
diff --git a/kernel-boot/rdma_topo b/kernel-boot/rdma_topo
index aa43faf..9b280a8 100755
--- a/kernel-boot/rdma_topo
+++ b/kernel-boot/rdma_topo
@@ -15,16 +15,17 @@
import subprocess
import sys
import tempfile
+import textwrap
from abc import ABC, abstractmethod
from base64 import b64encode, b64decode
+from enum import Enum
from typing import *
from zlib import compress, decompress
DEVDIR = os.environ.get("RDMA_TOPO_DEVDIR", "/sys/bus/pci/devices/")
BDF_RE = re.compile(r"^([0-9a-f]+?):([0-9a-f]{2}?):([0-9a-f]{2}?)\.([0-9a-f])$")
-KERNEL_ACS_ISOLATED = "xx111x1"
pci_vendors = {
"MELLANOX": 0x15B3,
"NVIDIA": 0x10DE,
@@ -41,10 +42,43 @@
class CommandError(Exception):
- pass
+ class Level(Enum):
+ ERROR = "E"
+ WARNING = "W"
+
+ def __init__(self, reason: str, level: Level = Level.ERROR):
+ self.reason = reason
+ self.level = level
+ super().__init__(f"{level.value}: {reason}")
+
+ def __str__(self):
+ return f"{self.level.value}: {self.reason}"
-TOPO_NOT_SUPPORTED = CommandError("No supported topology detected")
+class TopoNotSupportedError(CommandError):
+ def __init__(self, reason: str = ""):
+ details = ""
+ if len(reason) > 0:
+ details = f": {reason}"
+ super().__init__(f"No supported topology detected{details}",
+ CommandError.Level.ERROR)
+
+
+class TopoUnexpectedError(CommandError):
+ def __init__(self, reason: str = ""):
+ details = ""
+ if len(reason) > 0:
+ details = f": {reason}"
+ super().__init__(
+ (f"Unexpected topology structure{details}. Are you running on a "
+ "production system? If yes, please report this issue."),
+ CommandError.Level.ERROR,
+ )
+
+
+class DumpParsingError(CommandError):
+ def __init__(self, reason: str):
+ super().__init__(f"Malformed dump: {reason}", CommandError.Level.ERROR)
def yesno(b: bool) -> str:
@@ -146,7 +180,7 @@
try:
obj.data[k] = decompress(b64decode(obj.data[k]))
except Exception as e:
- raise ValueError(f"Invalid encoded value for key '{k}': {e}")
+ raise ValueError(f"Invalid encoded value for key '{k}': {e}") from e
return obj
@@ -162,13 +196,16 @@
return res
-def parse_vpd(vpd: Optional[bytes]) -> Tuple[Optional[str], Optional[str]]:
- """Parse VPD name and V3 UUID"""
+def parse_vpd(
+ vpd: Optional[bytes],
+) -> Tuple[Optional[str], Optional[str], Optional[str]]:
+ """Parse VPD name, V3 UUID, and serial number (SN)"""
if vpd is None:
- return None, None
+ return None, None, None
name = None
v3 = None
+ sn = None
def items(data: bytes) -> Generator[Tuple[int, bytes]]:
while len(data) > 0:
@@ -207,10 +244,12 @@
for keyword, value in keywords(item):
if keyword == "V3":
v3 = value.decode("ascii")
+ elif keyword == "SN":
+ sn = value.decode("ascii")
except UnicodeDecodeError:
pass
- return (v3, name)
+ return (v3, name, sn)
def parse_ext_cap(config: bytes, cap_id: int) -> Optional[bytes]:
@@ -248,6 +287,13 @@
return parse_ext_cap(config, PCI_EXT_CAP_ID_ATS) is not None
+def kernel_acs_isolated(device_type: str) -> str:
+ if device_type == "vera_rp":
+ # Advertises P2P Completion Redirect (bit-3) as read-only and disabled.
+ return "xx101x1"
+ return "xx111x1"
+
+
def PCI_VDEVICE(vendor: str, device_id: int) -> re.Pattern:
"""Match a Vendor and device ID"""
vendor_id = pci_vendors[vendor]
@@ -275,8 +321,11 @@
PCI_VDEVICE("NVIDIA", 0x22B1): "grace_rp", # NVIDIA Grace PCI Root Port Bridge
PCI_VDEVICE("NVIDIA", 0x22B2): "grace_rp", # NVIDIA Grace PCI Root Port Bridge
PCI_VDEVICE("NVIDIA", 0x22B8): "grace_rp", # NVIDIA Grace PCI Root Port Bridge
+ PCI_VDEVICE("NVIDIA", 0x2F95): "vera_rp", # NVIDIA Vera PCIe Root Port Bridge
+ PCI_VDEVICE("NVIDIA", 0x2F96): "vera_rp", # NVIDIA Vera PCIe Root Port Bridge
PCI_VDEVICE("MELLANOX", 0x1021): "cx_nic", # ConnectX-7
PCI_VDEVICE("MELLANOX", 0x1023): "cx_nic", # ConnectX-8
+ PCI_VDEVICE("MELLANOX", 0x1025): "cx_nic", # ConnectX-9
PCI_VDEVICE("MELLANOX", 0xA2DC): "bf3_nic", # BlueField-3
PCI_VDEVICE("MELLANOX", 0x2100): "cx_dma", # ConnectX-8 DMA Controller
PCI_VDEVICE("MELLANOX", 0x197B): "bf3_switch", # USP/DSP of a BF3 switch
@@ -324,6 +373,7 @@
device_type = ""
vpd_v3: Optional[str] = None
vpd_name: Optional[str] = None
+ vpd_sn: Optional[str] = None
parent: PCIDevice = None
def __init__(self, bdf: PCIBDF, sysfs_device: SysfsDevice):
@@ -350,7 +400,7 @@
def finish_loading(self):
"""Do more expensive parsing operations"""
if self.device_type == "cx_nic" or self.device_type == "cx_dma":
- self.vpd_v3, self.vpd_name = parse_vpd(self.sysfs_device.vpd)
+ self.vpd_v3, self.vpd_name, self.vpd_sn = parse_vpd(self.sysfs_device.vpd)
if "switch" in self.device_type or self.device_type.endswith("_rp"):
self.has_acs = self.get_acs_ctrl() is not None
if self.device_type == "cx_nic":
@@ -387,6 +437,135 @@
return self.sysfs_device.subsystems or {}
+class NVCX_Topo:
+
+ class NIC(object):
+ def __init__(self, pfs: Set[PCIDevice]):
+ assert len(pfs) > 0
+ self.pfs = pfs
+ self.parent = min(pfs, key=lambda pf: pf.bdf).parent
+ self.vpd_sn = min(pfs, key=lambda pf: pf.bdf).vpd_sn
+ if not all(pf.parent == self.parent for pf in pfs):
+ raise TopoUnexpectedError("All PFs of a NIC must have the same parent")
+ if not all(pf.vpd_sn == self.vpd_sn for pf in pfs):
+ raise TopoUnexpectedError("All PFs of a NIC must have the same VPD SN")
+
+ @property
+ def primary_pf(self) -> PCIDevice:
+ return min(self.pfs, key=lambda pf: pf.bdf)
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "parent_bdf": str(self.parent.bdf) if self.parent else "UNKNOWN",
+ "ats": self.primary_pf.has_ats,
+ "pf_bdfs": [
+ str(pf.bdf) for pf in sorted(self.pfs, key=lambda pf: pf.bdf)
+ ],
+ }
+
+ def str_single_pf(self) -> str:
+ res = ""
+ res += f"\tNIC ATS: {yesno(self.primary_pf.has_ats)}\n"
+ subsystems: Dict[str, Set[str]] = collections.defaultdict(set)
+ for pf in self.pfs:
+ for k, v in pf.get_subsystems().items():
+ subsystems[k].update(v)
+ res += print_list("RDMA device", subsystems["infiniband"])
+ res += print_list("Net device", subsystems["net"])
+ return res[:-1]
+
+ def __str__(self) -> str:
+ parent_bdf = self.parent.bdf if self.parent else "UNKNOWN"
+ res = f"RDMA NIC Parent={parent_bdf}\n"
+ res += print_list("NIC PCI device", [str(pf.bdf) for pf in self.pfs])
+ res += self.str_single_pf()
+ return res
+
+ class Board(object):
+ def __init__(self, nics: Set[NVCX_Topo.NIC]):
+ def nic_sort_key(nic):
+ if nic.parent:
+ return (0, nic.parent.bdf)
+ return (1, nic.primary_pf.bdf)
+
+ self.nics = sorted(nics, key=nic_sort_key)
+ self.sn = next(iter(nics)).vpd_sn
+ if not all(nic.vpd_sn == self.sn for nic in nics):
+ raise TopoUnexpectedError("All NICs of a Board must have the same VPD SN")
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "board_sn": self.sn or "UNKNOWN",
+ "nics": [nic.to_dict() for nic in self.nics],
+ }
+
+ def __str__(self) -> str:
+ board_sn = self.sn or "UNKNOWN"
+ res = f"RDMA NIC Board={board_sn}\n"
+ for nic in self.nics:
+ res += textwrap.indent(str(nic), "\t")
+ res += "\n"
+ return res[:-1]
+
+ def __init__(self, pfs: Set[PCIDevice]):
+ assert len(pfs) > 0
+
+ pfs_by_sn: Dict[Optional[str], Set[PCIDevice]] = collections.defaultdict(set)
+ for pf in pfs:
+ pfs_by_sn[pf.vpd_sn].add(pf)
+
+ boards = set()
+ for board_pfs in pfs_by_sn.values():
+ pfs_by_nic: Dict[Optional[PCIDevice], Set[PCIDevice]] = (
+ collections.defaultdict(set)
+ )
+ for pf in board_pfs:
+ pfs_by_nic[pf.parent].add(pf)
+
+ nics: Set[NVCX_Topo.NIC] = set()
+ for nic_pfs in pfs_by_nic.values():
+ nics.add(NVCX_Topo.NIC(nic_pfs))
+
+ boards.add(NVCX_Topo.Board(nics))
+
+ self.boards = sorted(boards, key=lambda b: b.sn or "")
+
+ @property
+ def primary_pf(self) -> PCIDevice:
+ return min(self.pfs, key=lambda pf: pf.bdf)
+
+ @property
+ def pfs(self) -> Set[PCIDevice]:
+ return set(pf for board in self.boards for nic in board.nics for pf in nic.pfs)
+
+ def to_dict(self) -> Dict[str, Any]:
+ if len(self.pfs) == 1:
+ return {
+ "rdma_nic_pf_bdf": str(next(iter(self.pfs)).bdf),
+ "rdma_nic_ats": next(iter(self.pfs)).has_ats,
+ }
+
+ return {
+ "rdma_nic_boards": [board.to_dict() for board in self.boards]
+ }
+
+ def topo_str_key(self) -> str:
+ if len(self.pfs) == 1:
+ return f"RDMA NIC={next(iter(self.pfs)).bdf}"
+
+ return ""
+
+ def topo_str(self) -> str:
+ if len(self.pfs) == 1:
+ return self.boards[0].nics[0].str_single_pf()
+
+ res = ""
+ for board in self.boards:
+ res += textwrap.indent(str(board), "\t")
+ res += "\n"
+ return res[:-1]
+
+
class NVCX_Complex(ABC):
@property
@abstractmethod
@@ -598,23 +777,23 @@
for pdev in dsp.iterdownstream():
if pdev.device_type == "cx_nic":
if self.cx_pf_dsp is not None:
- raise ValueError(
- f"Multiple CX NIC DSPs under the same shared switch not supported"
+ raise TopoNotSupportedError(
+ f"Multiple CX NIC DSPs under the same shared switch"
)
self.cx_pf_dsp = dsp
break
if pdev.device_type == "nvgpu":
if self.nvgpu_dsp is not None:
- raise ValueError(
- f"Multiple GPU DSPs under the same shared switch not supported"
+ raise TopoNotSupportedError(
+ f"Multiple GPU DSPs under the same shared switch"
)
self.nvgpu_dsp = dsp
break
if not self.cx_pf_dsp:
- raise ValueError(f"CX NIC DSP not found in the topology")
+ raise TopoUnexpectedError(f"CX NIC DSP not found in the topology")
if not self.nvgpu_dsp:
- raise ValueError(f"GPU DSP not found in the topology")
+ raise TopoUnexpectedError(f"GPU DSP not found in the topology")
@property
def primary_nic(self) -> PCIDevice:
@@ -622,13 +801,13 @@
def compute_acs(self, virt: Optional[bool]) -> Dict[PCIDevice, str]:
if not self.cx_pf_dsp.has_acs:
- raise CommandError(f"CX NIC DSP {self.cx_pf_dsp.bdf} lacks ACS")
+ raise TopoUnexpectedError(f"CX NIC DSP {self.cx_pf_dsp.bdf} lacks ACS")
if not self.nvgpu_dsp.has_acs:
- raise CommandError(f"GPU DSP {self.nvgpu_dsp.bdf} lacks ACS")
+ raise TopoUnexpectedError(f"GPU DSP {self.nvgpu_dsp.bdf} lacks ACS")
if not self.root_port.has_acs:
- raise CommandError(f"Root port {self.root_port.bdf} lacks ACS")
+ raise TopoUnexpectedError(f"Root port {self.root_port.bdf} lacks ACS")
if virt is None:
- raise CommandError("Unexpected: Could not determine virt mode")
+ raise TopoUnexpectedError("Could not determine virt mode")
if virt:
return {
@@ -646,8 +825,8 @@
# bit-3 : ACS P2P Completion Redirect
# bit-2 : ACS P2P Request Redirect
# bit-0 : ACS Source Validation
- self.nvgpu_dsp: KERNEL_ACS_ISOLATED,
- self.root_port: KERNEL_ACS_ISOLATED,
+ self.nvgpu_dsp: kernel_acs_isolated(self.nvgpu_dsp.device_type),
+ self.root_port: kernel_acs_isolated(self.root_port.device_type),
}
else:
return {
@@ -673,8 +852,7 @@
res["rdma_nic_vpd_name"] = self.cx_pf.vpd_name
if self.cx_pf.numa_node is not None:
res["numa_node"] = self.cx_pf.numa_node
- if self.cx_pf.has_ats:
- res["rdma_nic_ats"] = self.cx_pf.has_ats
+ res["rdma_nic_ats"] = self.cx_pf.has_ats
for pdev in sorted(
itertools.chain([self.cx_pf, self.nvgpu]),
@@ -761,6 +939,113 @@
return res[:-1]
+class NVCX_NUMA_Complex(NVCX_Complex):
+ def __init__(
+ self,
+ numa_node: int,
+ cx_pfs: Set[PCIDevice],
+ nvgpus: Set[PCIDevice],
+ ):
+ if not cx_pfs:
+ raise TopoUnexpectedError("No CX NICs found in NUMA-based complex")
+ if not nvgpus:
+ raise TopoUnexpectedError("No GPUs found in NUMA-based complex")
+ self.numa_node = numa_node
+ self.nvgpus = nvgpus
+ self.cx_topo = NVCX_Topo(cx_pfs)
+
+ @property
+ def primary_nic(self) -> PCIDevice:
+ return self.cx_topo.primary_pf
+
+ def compute_acs(self, _: Optional[bool]) -> Dict[PCIDevice, str]:
+ # All are connected to RPs, which are handled by PCITopo.compute_acs
+ return {}
+
+ def to_dict(self) -> Dict[str, Any]:
+ res = {
+ "numa_node": self.numa_node,
+ "gpu_bdfs": [str(I.bdf) for I in self.nvgpus],
+ "subsystems": {},
+ }
+ res.update(self.cx_topo.to_dict())
+ devname = self.cx_topo.primary_pf.vpd_name
+ if devname:
+ res["rdma_nic_vpd_name"] = devname
+ for pdev in sorted(
+ itertools.chain(self.cx_topo.pfs, self.nvgpus),
+ key=lambda x: x.bdf,
+ ):
+ subsystems = pdev.get_subsystems()
+ if subsystems:
+ res["subsystems"][str(pdev.bdf)] = {
+ subsys: list(devs) for subsys, devs in subsystems.items()
+ }
+ return res
+
+ def __check_iommu_group(self) -> bool:
+ devs = [*self.cx_topo.pfs, *self.nvgpus]
+ iommu_groups = [dev.iommu_group for dev in devs]
+ devs_without_iommu_group = [dev for dev in devs if dev.iommu_group is None]
+ if len(devs_without_iommu_group) > 0:
+ bdfs = ", ".join([str(dev.bdf) for dev in devs_without_iommu_group])
+ check_fail(
+ f"Kernel iommu_group missing for devices on NUMA node {self.numa_node}: {bdfs}"
+ )
+ return False
+ if len(iommu_groups) == len(set(iommu_groups)):
+ check_ok(
+ f"All kernel iommu_groups for NUMA node {self.numa_node} are unique"
+ )
+ return True
+
+ duplicate_groups = [
+ group for group in iommu_groups if iommu_groups.count(group) > 1
+ ]
+ check_fail(
+ (f"Multiple devices share the same kernel iommu_group for "
+ f"NUMA node {self.numa_node}: {duplicate_groups}")
+ )
+ return False
+
+ def __check_ats(self) -> bool:
+ result = True
+ for cx_pf in self.cx_topo.pfs:
+ if cx_pf.has_ats:
+ check_fail(
+ f"ATS capability for {cx_pf.device_type} {cx_pf.bdf} is available"
+ )
+ result = False
+ else:
+ check_ok(
+ f"ATS capability for {cx_pf.device_type} {cx_pf.bdf} is not available"
+ )
+ return result
+
+ def check(self, virt: Optional[bool]) -> bool:
+ res_ats = self.__check_ats()
+ res_iommu_group = self.__check_iommu_group()
+ return res_ats and res_iommu_group
+
+ def __str__(self) -> str:
+ res = f"NUMA Node={self.numa_node}\n"
+ devname = self.primary_nic.vpd_name
+ if devname:
+ res += f"\t{devname}\n"
+ res += f"{self.cx_topo.topo_str()}\n"
+ res += print_list("GPU PCI device", [str(I.bdf) for I in self.nvgpus])
+
+ gpu_subsystems: Set[str] = set()
+ for pdev in self.nvgpus:
+ subsys = pdev.get_subsystems()
+ if "drm" in subsys:
+ gpu_subsystems.update(subsys["drm"])
+ if len(gpu_subsystems) > 0:
+ res += print_list("DRM device", gpu_subsystems)
+
+ return res[:-1]
+
+
def check_parent(pdev: PCIDevice, parent_type: str):
if not pdev or not pdev.parent:
return None
@@ -771,6 +1056,10 @@
class PCITopo(object):
"""Load the PCI topology from sysfs and organize it"""
+ class TopoType(Enum):
+ INLINE = "Inline"
+ DMA = "DMA-based"
+ NUMA = "NUMA-based"
def __init__(
self,
@@ -784,37 +1073,45 @@
sysfs_devices = [SysfsDevice(fn) for fn in os.listdir(DEVDIR)]
self.devices = self.__load_devices(sysfs_devices)
self.nvcxs: List[NVCX_Complex] = []
- self.has_cx_dma = any(
- pdev.device_type == "cx_dma" for pdev in self.devices.values()
- )
- self.has_gpu_and_nic = False
+ self.type = self.__detect_topo_type()
- if self.has_cx_dma and virt is not None:
+ if self.type == self.TopoType.DMA and virt is not None:
raise CommandError(
- "--virt / --no-virt is not supported on DMA-based topologies"
+ f"--virt / --no-virt is not supported on {self.type.value} topologies"
)
self.virt = virt
self._autodetect_virt = autodetect_virt
- if not self.has_cx_dma:
- found = {
- "cx_switch": False,
- "nvgpu": False,
- "cx_nic": False,
- }
- for pdev in self.devices.values():
- if pdev.device_type not in found.keys():
- continue
- found[pdev.device_type] = True
- self.has_gpu_and_nic = all(found.values())
-
- if not self.has_gpu_and_nic:
- return
-
for pdev in self.devices.values():
pdev.finish_loading()
self.__build_topo()
+ def __detect_topo_type(self) -> TopoType:
+ if any(pdev.device_type == "cx_dma" for pdev in self.devices.values()):
+ return self.TopoType.DMA
+
+ if any(
+ pdev.device_type == "nvgpu"
+ and pdev.parent is not None
+ and pdev.parent.device_type.endswith("_rp")
+ for pdev in self.devices.values()
+ ):
+ return self.TopoType.NUMA
+
+ found = {
+ "cx_switch": False,
+ "nvgpu": False,
+ "cx_nic": False,
+ }
+ for pdev in self.devices.values():
+ if pdev.device_type not in found.keys():
+ continue
+ found[pdev.device_type] = True
+ if all(found.values()):
+ return self.TopoType.INLINE
+
+ raise TopoNotSupportedError()
+
def __parse_dump(self, filename: str) -> List[SysfsDevice]:
res: List[SysfsDevice] = []
try:
@@ -836,7 +1133,7 @@
raise ValueError(f"Item {i}/{num_items}: {e}") from e
return res
except (json.JSONDecodeError, ValueError) as e:
- raise CommandError(f"Invalid sysfs dump file: {e}")
+ raise DumpParsingError(str(e))
except (FileNotFoundError, PermissionError) as e:
raise CommandError(f"Failed to read sysfs dump file: {e}")
@@ -848,6 +1145,10 @@
continue
assert bdf not in res
res[bdf] = PCIDevice(bdf, sdev)
+ for pdev in res.values():
+ if pdev.parent_bdf and pdev.parent_bdf in res:
+ pdev.parent = res[pdev.parent_bdf]
+ pdev.parent.children.add(pdev)
return res
def __get_nvcx_complex(self, cx_dma: PCIDevice):
@@ -859,22 +1160,21 @@
"""
assert cx_dma.device_type == "cx_dma"
if not cx_dma.vpd_v3:
- raise ValueError(f"CX DMA function {cx_dma} does not have a VPD V3 UUID")
+ raise TopoUnexpectedError(f"CX DMA function {cx_dma} does not have a VPD V3 UUID")
# The DMA and PF are matched using the UUID from the VPD
cx_pfs = self.vpd_v3s.get(cx_dma.vpd_v3)
if cx_pfs is None:
- raise ValueError(
+ raise TopoUnexpectedError(
f"CX DMA function {cx_dma} does not have a matching PF, V3 UUID matching failed"
)
- return None
# Path from the DMA to the root port
cx_dma_dsp = check_parent(cx_dma, "cx_switch")
cx_usp = check_parent(cx_dma_dsp, "cx_switch")
grace_rp = check_parent(cx_usp, "grace_rp")
if not grace_rp:
- raise ValueError(
+ raise TopoUnexpectedError(
f"CX DMA function {cx_dma} has an unrecognized upstream path"
)
@@ -883,13 +1183,13 @@
pdev for pdev in grace_rp.iterdownstream() if pdev.device_type == "nvgpu"
]
if len(nvgpus) != 1:
- raise ValueError(f"CX DMA function {cx_dma} does not have a nearby GPU")
+ raise TopoUnexpectedError(f"CX DMA function {cx_dma} does not have a nearby GPU")
nvgpu = nvgpus[0]
nvgpu_dsp2 = check_parent(nvgpu, "cx_switch")
nvgpu_usp2 = check_parent(nvgpu_dsp2, "cx_switch")
nvgpu_dsp1 = check_parent(nvgpu_usp2, "cx_switch")
if cx_usp != check_parent(nvgpu_dsp1, "cx_switch"):
- raise ValueError(
+ raise TopoNotSupportedError(
f"CX DMA function {cx_dma} has an unrecognized upstream path from the GPU"
)
@@ -905,7 +1205,7 @@
}
topodevs = set(grace_rp.iterdownstream())
if alldevs != topodevs:
- raise ValueError(
+ raise TopoUnexpectedError(
f"CX DMA function {cx_dma} has unexpected PCI devices in the topology"
)
@@ -924,14 +1224,16 @@
nvgpu_dsp1 = check_parent(nvgpu_usp2, "cx_switch")
shared_usp1 = check_parent(nvgpu_dsp1, "cx_switch")
if not shared_usp1:
- raise ValueError(f"GPU {nvgpu} has an unrecognized upstream path")
+ raise TopoNotSupportedError(
+ f"GPU {nvgpu.bdf} has an unrecognized upstream path"
+ )
for pdev in shared_usp1.iterupstream_path():
if pdev.device_type == "generic_rp":
root_port = pdev
break
else:
- raise ValueError(
+ raise TopoUnexpectedError(
f"Could not find root port for shared USP {shared_usp1.bdf}"
)
@@ -940,16 +1242,34 @@
cx_nic = pdev
break
else:
- raise ValueError(f"GPU {nvgpu} does not have a nearby CX NIC")
+ raise TopoUnexpectedError(f"GPU {nvgpu} does not have a nearby CX NIC")
return NVCX_Inline_Complex(root_port, shared_usp1, cx_nic, nvgpu)
+ def __get_nvcx_numa_complex(self, numa_node: int):
+ """Match the topology for the NUMA complex.
+
+ All CX NICs and GPUs on the same NUMA node are part of the same complex.
+ """
+ cx_pfs = [
+ pdev
+ for pdev in self.devices.values()
+ if pdev.numa_node == numa_node and pdev.device_type == "cx_nic"
+ ]
+ nvgpus = [
+ pdev
+ for pdev in self.devices.values()
+ if pdev.numa_node == numa_node and pdev.device_type == "nvgpu"
+ ]
+ return NVCX_NUMA_Complex(numa_node, cx_pfs, nvgpus)
+
def __auto_detect_virt(self) -> bool:
"""Auto-detect if virtualization will be used on this system"""
first = self.nvcxs[0].primary_nic.has_ats
if not all(nvcx.primary_nic.has_ats == first for nvcx in self.nvcxs):
- raise CommandError(
- "Could not auto-detect virtualization: CX NICs have different ATS settings"
+ raise TopoNotSupportedError(
+ "Could not auto-detect virtualization. CX NICs have different"
+ " ATS settings. Try explicitly setting --virt or --no-virt."
)
return first
@@ -959,36 +1279,40 @@
objects for the cx_dma functions"""
self.vpd_v3s: Dict[str, Set[PCIDevice]] = collections.defaultdict(set)
for pdev in self.devices.values():
- if pdev.parent_bdf and pdev.parent_bdf in self.devices:
- pdev.parent = self.devices[pdev.parent_bdf]
- pdev.parent.children.add(pdev)
-
# Many PCI functions may share the same V3
if pdev.vpd_v3:
self.vpd_v3s[pdev.vpd_v3].add(pdev)
- if self.has_cx_dma:
+ if self.type == self.TopoType.DMA:
for pdev in self.devices.values():
if pdev.device_type == "cx_dma":
nvcx = self.__get_nvcx_complex(pdev)
self.nvcxs.append(nvcx)
- elif self.has_gpu_and_nic:
+ elif self.type == self.TopoType.INLINE:
for pdev in self.devices.values():
if pdev.device_type == "nvgpu":
nvcx = self.__get_nvcx_inline_complex(pdev)
self.nvcxs.append(nvcx)
+ elif self.type == self.TopoType.NUMA:
+ numa_nodes = {
+ pdev.numa_node
+ for pdev in self.devices.values()
+ if pdev.numa_node is not None and pdev.numa_node >= 0
+ }
+ for numa in numa_nodes:
+ self.nvcxs.append(self.__get_nvcx_numa_complex(numa))
- if self.has_gpu_and_nic and len(self.nvcxs) > 0:
+ if len(self.nvcxs) == 0:
+ raise TopoNotSupportedError(
+ f"No supported complex found for {self.type.value} topology"
+ )
+
+ if self.type == self.TopoType.INLINE:
if self.virt is None and self._autodetect_virt:
self.virt = self.__auto_detect_virt()
self.nvcxs.sort(key=lambda x: x.primary_nic.bdf)
- @property
- def supported(self) -> bool:
- """True if the system has a topology that is supported by the rdma_topo tool"""
- return (self.has_cx_dma or self.has_gpu_and_nic) and len(self.nvcxs) > 0
-
def compute_acs(self):
"""Return a dictionary of PCI devices and the ACS mask the device should
have"""
@@ -997,8 +1321,8 @@
acs.update(nvcx.compute_acs(self.virt))
# Enable, using kernel default, or disable ACS on all other CX
- # bridges and Grace RP based on the virt parameter or if the topology
- # has CX DMA functions.
+ # bridges and Grace, Vera RPs based on the virt parameter or if the
+ # topology is DMA-based or NUMA-based.
#
# To enable (matches kernel default):
# bit-4 : ACS Upstream Forwarding
@@ -1008,11 +1332,16 @@
for pdev in self.devices.values():
if (
pdev not in acs
- and ("switch" in pdev.device_type or "grace_rp" in pdev.device_type)
+ and (
+ "switch" in pdev.device_type
+ or pdev.device_type in ["grace_rp", "vera_rp"]
+ )
and pdev.has_acs
):
acs[pdev] = (
- KERNEL_ACS_ISOLATED if self.has_cx_dma or self.virt else "xx000x0"
+ kernel_acs_isolated(pdev.device_type)
+ if self.type in [self.TopoType.DMA, self.TopoType.NUMA] or self.virt
+ else "xx000x0"
)
return acs
@@ -1072,8 +1401,6 @@
"""List the ConnectX NICs in the system with the corresponding NIC
function, associated GPU, and, optionally, DMA Direct function."""
topo = PCITopo(args.sysfs_dump, virt=None, autodetect_virt=False)
- if not topo.supported:
- raise TOPO_NOT_SUPPORTED
if args.json:
return topo_json(topo)
@@ -1131,22 +1458,25 @@
If the system does not have any need of ACS flags the dropin file will be
removed. This command is intended for Debian style systems with a
/etc/default/grub.d and update-grub command."""
- topo = PCITopo(None, args.virt)
- if not topo.supported:
+ try:
+ topo = PCITopo(None, args.virt)
+ except TopoNotSupportedError as e:
if args.dry_run:
- raise TOPO_NOT_SUPPORTED
+ raise
if os.path.exists(args.output):
- print(
- f"W: Found ACS drop-in file {args.output} but the system does not have a supported topology. Deleting file."
- )
os.unlink(args.output)
+ raise CommandError(
+ (f"Found ACS drop-in file {args.output} but the system does "
+ "not have a supported topology. File deleted."),
+ CommandError.Level.WARNING,
+ )
return
acs = topo.compute_acs()
config_acs = [
f"{acs}@{pdev.bdf}"
for pdev, acs in sorted(acs.items(), key=lambda x: x[0].bdf)
- if acs != KERNEL_ACS_ISOLATED
+ if acs != kernel_acs_isolated(pdev.device_type)
]
acs_arg = ";".join(config_acs)
grub_conf = [
@@ -1199,8 +1529,6 @@
failures, use with caution!
"""
topo = PCITopo(None, args.virt)
- if not topo.supported:
- raise TOPO_NOT_SUPPORTED
acs = topo.compute_acs()
cmds: List[List[str]] = []
for pdev, acs in sorted(acs.items(), key=lambda x: x[0].bdf):
@@ -1241,12 +1569,12 @@
"""Check that the running kernel and PCI environment are setup correctly for
GPU Direct with ConnectX DMA Direct PCI functions."""
topo = PCITopo(args.sysfs_dump, args.virt)
- if not topo.supported:
- raise TOPO_NOT_SUPPORTED
- if topo.has_cx_dma:
+ if topo.type == PCITopo.TopoType.DMA:
check_ok("All ConnectX DMA functions have correct PCI topology")
- elif topo.has_gpu_and_nic:
+ elif topo.type == PCITopo.TopoType.INLINE:
check_ok("All NIC/GPU complexes have correct PCI topology")
+ elif topo.type == PCITopo.TopoType.NUMA:
+ check_ok("All NUMA-based complexes have correct PCI topology")
fatal = False
acs = topo.compute_acs()
@@ -1351,8 +1679,8 @@
try:
args.func(args)
except CommandError as e:
- print(f"E: {e}")
- sys.exit(100)
+ print(e)
+ sys.exit(100 if e.level == CommandError.Level.ERROR else 0)
main()
diff --git a/kernel-boot/rdma_topo_unit_test.py b/kernel-boot/rdma_topo_unit_test.py
new file mode 100644
index 0000000..3482023
--- /dev/null
+++ b/kernel-boot/rdma_topo_unit_test.py
@@ -0,0 +1,611 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Linux-OpenIB
+# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES
+"""Unit tests for NVCX_Topo and its nested NIC / Board types.
+
+Tests import rdma_topo via load_rdma_topo() (which strips the bare main()
+call) and use lightweight mock PCIDevice objects so no sysfs or dump file
+is needed.
+"""
+from __future__ import annotations
+
+import sys
+import types
+
+from pathlib import Path
+from typing import Dict, List, Optional
+from unittest.mock import MagicMock
+
+try:
+ import pytest
+except ImportError:
+ print("Missing dependency: pytest", file=sys.stderr)
+ print("Install with: pip3 install pytest", file=sys.stderr)
+ sys.exit(1)
+
+HERE = Path(__file__).resolve().parent
+RDMA_TOPO = HERE / "rdma_topo"
+
+
+def _strip_trailing_main_call(src: str) -> str:
+ """rdma_topo ends with bare main(); skip it so test import does not run CLI."""
+ lines = src.splitlines()
+ i = len(lines) - 1
+ while i >= 0 and lines[i].strip() == "":
+ i -= 1
+ if i < 0:
+ return src
+ if lines[i].split("#", 1)[0].strip() == "main()":
+ return "\n".join(lines[:i]) + ("\n" if i else "")
+ return src
+
+
+def load_rdma_topo():
+ raw = RDMA_TOPO.read_text(encoding="utf-8")
+ code_s = _strip_trailing_main_call(raw)
+ mod = types.ModuleType("rdma_topo")
+ mod.__file__ = str(RDMA_TOPO)
+ mod.__name__ = "rdma_topo"
+ mod.__package__ = ""
+ sys.modules["rdma_topo"] = mod
+ exec(compile(code_s, str(RDMA_TOPO), "exec"), mod.__dict__)
+ return mod
+
+
+load_rdma_topo()
+
+from rdma_topo import PCIBDF, NVCX_Topo, TopoUnexpectedError
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def make_dev(
+ bdf_str: str,
+ vpd_sn: Optional[str] = None,
+ parent=None,
+ has_ats: bool = False,
+ subsystems: Optional[Dict] = None,
+) -> MagicMock:
+ """Return a mock PCIDevice for use with NVCX_Topo code."""
+ dev = MagicMock(name=f"PCIDevice({bdf_str})")
+ seg, bus, rest = bdf_str.split(":")
+ d, func = rest.split(".")
+ dev.bdf = PCIBDF(seg, bus, d, func)
+ dev.vpd_sn = vpd_sn
+ dev.parent = parent
+ dev.has_ats = has_ats
+ dev.get_subsystems.return_value = subsystems or {}
+ return dev
+
+
+def make_parent(bdf_str: str) -> MagicMock:
+ """Return a mock parent PCIDevice (used as NIC.parent)."""
+ p = MagicMock(name=f"ParentDevice({bdf_str})")
+ seg, bus, rest = bdf_str.split(":")
+ d, func = rest.split(".")
+ p.bdf = PCIBDF(seg, bus, d, func)
+ return p
+
+
+def make_nic(
+ pf_bdfs: List[str],
+ parent_bdf: Optional[str] = None,
+ vpd_sn: Optional[str] = None,
+ has_ats: bool = False,
+ subsystems: Optional[Dict] = None,
+) -> NVCX_Topo.NIC:
+ """Construct a NVCX_Topo.NIC from mock PCIDevices."""
+ parent = make_parent(parent_bdf) if parent_bdf else None
+ devs = [
+ make_dev(
+ bdf, vpd_sn=vpd_sn, parent=parent, has_ats=has_ats, subsystems=subsystems
+ )
+ for bdf in pf_bdfs
+ ]
+ return NVCX_Topo.NIC(set(devs))
+
+
+# ---------------------------------------------------------------------------
+# NVCX_Topo.NIC — constructor
+# ---------------------------------------------------------------------------
+
+
+class TestNVCX_Topo_NIC_Constructor:
+ def test_single_pf_no_parent(self):
+ nic = make_nic(["0000:00:01.0"], parent_bdf=None, vpd_sn=None)
+ assert nic.parent is None
+ assert nic.vpd_sn is None
+ assert len(nic.pfs) == 1
+
+ def test_single_pf_with_parent_and_sn(self):
+ nic = make_nic(["0000:00:01.0"], parent_bdf="0000:00:00.0", vpd_sn="SN123")
+ assert str(nic.parent.bdf) == "0000:00:00.0"
+ assert nic.vpd_sn == "SN123"
+
+ def test_multi_pf_same_parent_same_sn(self):
+ nic = make_nic(
+ ["0000:00:01.0", "0000:00:02.0"],
+ parent_bdf="0000:00:00.0",
+ vpd_sn="SN123",
+ )
+ assert len(nic.pfs) == 2
+
+ def test_multi_pf_different_parent_raises(self):
+ parent_a = make_parent("0000:00:00.0")
+ parent_b = make_parent("0000:00:10.0")
+ dev_a = make_dev("0000:00:01.0", parent=parent_a, vpd_sn="SN")
+ dev_b = make_dev("0000:00:02.0", parent=parent_b, vpd_sn="SN")
+ with pytest.raises(TopoUnexpectedError, match="same parent"):
+ NVCX_Topo.NIC({dev_a, dev_b})
+
+ def test_multi_pf_different_sn_raises(self):
+ parent = make_parent("0000:00:00.0")
+ dev_a = make_dev("0000:00:01.0", parent=parent, vpd_sn="SN1")
+ dev_b = make_dev("0000:00:02.0", parent=parent, vpd_sn="SN2")
+ with pytest.raises(TopoUnexpectedError, match="same VPD SN"):
+ NVCX_Topo.NIC({dev_a, dev_b})
+
+
+# ---------------------------------------------------------------------------
+# NVCX_Topo.NIC — primary_pf
+# ---------------------------------------------------------------------------
+
+
+class TestNVCX_Topo_NIC_PrimaryPf:
+ def test_multi_pf_returns_min_bdf(self):
+ parent = make_parent("0000:00:00.0")
+ dev_lo = make_dev("0000:00:01.0", parent=parent, vpd_sn="SN")
+ dev_hi = make_dev("0000:00:02.0", parent=parent, vpd_sn="SN")
+ nic = NVCX_Topo.NIC({dev_lo, dev_hi})
+ assert nic.primary_pf is dev_lo
+
+
+# ---------------------------------------------------------------------------
+# NVCX_Topo.NIC — to_dict
+# ---------------------------------------------------------------------------
+
+
+class TestNVCX_Topo_NIC_ToDict:
+ def test_has_parent_bdf(self):
+ nic = make_nic(["0000:00:01.0"], parent_bdf="0000:00:00.0", vpd_sn="SN123")
+ assert nic.to_dict()["parent_bdf"] == "0000:00:00.0"
+
+ def test_no_parent_gives_unknown(self):
+ nic = make_nic(["0000:00:01.0"], parent_bdf=None, vpd_sn=None)
+ assert nic.to_dict()["parent_bdf"] == "UNKNOWN"
+
+
+ def test_ats_taken_from_min_bdf_pf(self):
+ parent = make_parent("0000:00:00.0")
+ dev_lo = make_dev("0000:00:01.0", parent=parent, vpd_sn="SN", has_ats=False)
+ dev_hi = make_dev("0000:00:02.0", parent=parent, vpd_sn="SN", has_ats=True)
+ nic = NVCX_Topo.NIC({dev_lo, dev_hi})
+ assert nic.to_dict()["ats"] == False
+ nic = NVCX_Topo.NIC({dev_hi, dev_lo})
+ assert nic.to_dict()["ats"] == False
+
+
+# ---------------------------------------------------------------------------
+# NVCX_Topo.NIC — __str__
+# ---------------------------------------------------------------------------
+
+
+class TestNVCX_Topo_NIC_Str:
+ def test_single_pf_starts_with_nic_parent_header(self):
+ nic = make_nic(["0000:00:01.0"], parent_bdf="0000:00:00.0", vpd_sn="SN123")
+ assert str(nic).startswith("RDMA NIC Parent=0000:00:00.0")
+
+ def test_single_pf_no_parent_shows_unknown(self):
+ nic = make_nic(["0000:00:01.0"], parent_bdf=None, vpd_sn=None)
+ assert str(nic).startswith("RDMA NIC Parent=UNKNOWN")
+
+ def test_single_pf_ats_no(self):
+ nic = make_nic(["0000:00:01.0"], parent_bdf="0000:00:00.0", vpd_sn="SN123")
+ assert "NIC ATS: no" in str(nic)
+
+ def test_single_pf_ats_yes(self):
+ nic = make_nic(
+ ["0000:00:01.0"],
+ parent_bdf="0000:00:00.0",
+ vpd_sn="SN123",
+ has_ats=True,
+ )
+ assert "NIC ATS: yes" in str(nic)
+
+ def test_no_trailing_newline(self):
+ nic = make_nic(["0000:00:01.0"], parent_bdf="0000:00:00.0", vpd_sn="SN123")
+ assert not str(nic).endswith("\n")
+
+ def test_single_pf_with_infiniband(self):
+ nic = make_nic(
+ ["0000:00:01.0"],
+ parent_bdf="0000:00:00.0",
+ vpd_sn="SN123",
+ subsystems={"infiniband": {"mlx5_0"}},
+ )
+ assert "RDMA device: mlx5_0" in str(nic)
+
+ def test_single_pf_with_net(self):
+ nic = make_nic(
+ ["0000:00:01.0"],
+ parent_bdf="0000:00:00.0",
+ vpd_sn="SN123",
+ subsystems={"net": {"eth0"}},
+ )
+ assert "Net device: eth0" in str(nic)
+
+ def test_multi_pf_shows_sorted_pci_device_list(self):
+ nic = make_nic(
+ ["0000:00:02.0", "0000:00:01.0"],
+ parent_bdf="0000:00:00.0",
+ vpd_sn="SN123",
+ )
+ assert "NIC PCI devices: 0000:00:01.0, 0000:00:02.0" in str(nic)
+
+ def test_multi_pf_subsystems_merged(self):
+ parent = make_parent("0000:00:00.0")
+ dev1 = make_dev(
+ "0000:00:01.0",
+ parent=parent,
+ vpd_sn="SN",
+ subsystems={"infiniband": {"mlx5_0"}, "net": {"eth0"}},
+ )
+ dev2 = make_dev(
+ "0000:00:02.0",
+ parent=parent,
+ vpd_sn="SN",
+ subsystems={"infiniband": {"mlx5_1"}, "net": {"eth1"}},
+ )
+ nic = NVCX_Topo.NIC({dev1, dev2})
+ result = str(nic)
+ assert "RDMA devices: mlx5_0, mlx5_1" in result
+ assert "Net devices: eth0, eth1" in result
+
+
+# ---------------------------------------------------------------------------
+# NVCX_Topo.Board — constructor
+# ---------------------------------------------------------------------------
+
+
+class TestNVCX_Topo_Board_Constructor:
+ def test_single_nic_sn_stored(self):
+ nic = make_nic(["0000:00:01.0"], parent_bdf="0000:00:00.0", vpd_sn="SN123")
+ board = NVCX_Topo.Board({nic})
+ assert board.sn == "SN123"
+ assert nic in board.nics
+
+ def test_multi_nic_same_sn(self):
+ nic_a = make_nic(["0000:00:01.0"], parent_bdf="0000:00:00.0", vpd_sn="SN123")
+ nic_b = make_nic(["0000:00:11.0"], parent_bdf="0000:00:10.0", vpd_sn="SN123")
+ board = NVCX_Topo.Board({nic_a, nic_b})
+ assert len(board.nics) == 2
+ assert board.sn == "SN123"
+
+ def test_multi_nic_different_sn_raises(self):
+ nic_a = make_nic(["0000:00:01.0"], parent_bdf="0000:00:00.0", vpd_sn="SN1")
+ nic_b = make_nic(["0000:00:11.0"], parent_bdf="0000:00:10.0", vpd_sn="SN2")
+ with pytest.raises(TopoUnexpectedError, match="same VPD SN"):
+ NVCX_Topo.Board({nic_a, nic_b})
+
+
+# ---------------------------------------------------------------------------
+# NVCX_Topo.Board — to_dict
+# ---------------------------------------------------------------------------
+
+
+class TestNVCX_Topo_Board_ToDict:
+ def test_has_board_sn(self):
+ nic = make_nic(["0000:00:01.0"], parent_bdf="0000:00:00.0", vpd_sn="SN123")
+ board = NVCX_Topo.Board({nic})
+ assert board.to_dict()["board_sn"] == "SN123"
+
+ def test_none_sn_gives_unknown(self):
+ nic = make_nic(["0000:00:01.0"], parent_bdf=None, vpd_sn=None)
+ board = NVCX_Topo.Board({nic})
+ assert board.to_dict()["board_sn"] == "UNKNOWN"
+
+ def test_nics_list_length(self):
+ nic_a = make_nic(["0000:00:01.0"], parent_bdf="0000:00:00.0", vpd_sn="SN123")
+ nic_b = make_nic(["0000:00:11.0"], parent_bdf="0000:00:10.0", vpd_sn="SN123")
+ board = NVCX_Topo.Board({nic_a, nic_b})
+ assert len(board.to_dict()["nics"]) == 2
+
+ def test_nics_list_contains_nic_dicts(self):
+ nic = make_nic(
+ ["0000:00:01.0", "0000:00:02.0"],
+ parent_bdf="0000:00:00.0",
+ vpd_sn="SN123",
+ )
+ board = NVCX_Topo.Board({nic})
+ nic_dict = board.to_dict()["nics"][0]
+ assert nic_dict == nic.to_dict()
+
+
+# ---------------------------------------------------------------------------
+# NVCX_Topo.Board — __str__
+# ---------------------------------------------------------------------------
+
+
+class TestNVCX_Topo_Board_Str:
+ def test_single_nic_starts_with_board_header(self):
+ nic = make_nic(["0000:00:01.0"], parent_bdf="0000:00:00.0", vpd_sn="SN123")
+ board = NVCX_Topo.Board({nic})
+ assert str(board).startswith("RDMA NIC Board=SN123")
+
+ def test_none_sn_shows_unknown(self):
+ nic = make_nic(["0000:00:01.0"], parent_bdf=None, vpd_sn=None)
+ board = NVCX_Topo.Board({nic})
+ assert str(board).startswith("RDMA NIC Board=UNKNOWN")
+
+ def test_multi_nic_includes_nic_parent_headers(self):
+ nic_a = make_nic(["0000:00:01.0"], parent_bdf="0000:00:00.0", vpd_sn="SN123")
+ nic_b = make_nic(["0000:00:11.0"], parent_bdf="0000:00:10.0", vpd_sn="SN123")
+ board = NVCX_Topo.Board({nic_a, nic_b})
+ result = str(board)
+ assert "RDMA NIC Parent=0000:00:00.0" in result
+ assert "RDMA NIC Parent=0000:00:10.0" in result
+
+ def test_multi_nic_nic_body_indented(self):
+ nic_a = make_nic(["0000:00:01.0"], parent_bdf="0000:00:00.0", vpd_sn="SN123")
+ nic_b = make_nic(["0000:00:11.0"], parent_bdf="0000:00:10.0", vpd_sn="SN123")
+ board = NVCX_Topo.Board({nic_a, nic_b})
+ result = str(board)
+ assert "\t\tNIC ATS: no" in result
+
+ def test_multi_nic_no_trailing_newline(self):
+ nic_a = make_nic(["0000:00:01.0"], parent_bdf="0000:00:00.0", vpd_sn="SN123")
+ nic_b = make_nic(["0000:00:11.0"], parent_bdf="0000:00:10.0", vpd_sn="SN123")
+ board = NVCX_Topo.Board({nic_a, nic_b})
+ assert not str(board).endswith("\n")
+
+
+# ---------------------------------------------------------------------------
+# NVCX_Topo — constructor
+# ---------------------------------------------------------------------------
+
+
+class TestNVCX_Topo_Constructor:
+ def test_single_pf_with_parent_and_sn_yields_one_board(self):
+ parent = make_parent("0000:00:00.0")
+ dev = make_dev("0000:00:01.0", vpd_sn="SN123", parent=parent)
+ topo = NVCX_Topo({dev})
+ assert len(topo.boards) == 1
+
+ def test_two_pfs_same_parent_same_sn_one_nic_one_board(self):
+ parent = make_parent("0000:00:00.0")
+ dev1 = make_dev("0000:00:01.0", vpd_sn="SN123", parent=parent)
+ dev2 = make_dev("0000:00:02.0", vpd_sn="SN123", parent=parent)
+ topo = NVCX_Topo({dev1, dev2})
+ assert len(topo.boards) == 1
+ assert len(topo.boards[0].nics) == 1
+ assert len(topo.boards[0].nics[0].pfs) == 2
+
+ def test_two_pfs_different_parent_same_sn_two_nics_one_board(self):
+ parent_a = make_parent("0000:00:00.0")
+ parent_b = make_parent("0000:00:10.0")
+ dev1 = make_dev("0000:00:01.0", vpd_sn="SN123", parent=parent_a)
+ dev2 = make_dev("0000:00:11.0", vpd_sn="SN123", parent=parent_b)
+ topo = NVCX_Topo({dev1, dev2})
+ assert len(topo.boards) == 1
+ assert len(topo.boards[0].nics) == 2
+
+ def test_two_pfs_different_sn_two_boards(self):
+ parent_a = make_parent("0000:00:00.0")
+ parent_b = make_parent("0000:00:10.0")
+ dev1 = make_dev("0000:00:01.0", vpd_sn="SN1", parent=parent_a)
+ dev2 = make_dev("0000:00:11.0", vpd_sn="SN2", parent=parent_b)
+ topo = NVCX_Topo({dev1, dev2})
+ assert len(topo.boards) == 2
+
+
+# ---------------------------------------------------------------------------
+# NVCX_Topo — pfs / primary_pf
+# ---------------------------------------------------------------------------
+
+
+class TestNVCX_Topo_Pfs:
+ def test_pfs_returns_all_devs(self):
+ parent = make_parent("0000:00:00.0")
+ dev1 = make_dev("0000:00:01.0", vpd_sn="SN123", parent=parent)
+ dev2 = make_dev("0000:00:02.0", vpd_sn="SN123", parent=parent)
+ topo = NVCX_Topo({dev1, dev2})
+ assert topo.pfs == {dev1, dev2}
+
+ def test_primary_pf_is_min_bdf(self):
+ parent = make_parent("0000:00:00.0")
+ dev_lo = make_dev("0000:00:01.0", vpd_sn="SN123", parent=parent)
+ dev_hi = make_dev("0000:00:02.0", vpd_sn="SN123", parent=parent)
+ topo = NVCX_Topo({dev_lo, dev_hi})
+ assert topo.primary_pf is dev_lo
+
+ def test_primary_pf_across_boards(self):
+ parent_a = make_parent("0000:00:00.0")
+ parent_b = make_parent("0000:00:10.0")
+ dev_lo = make_dev("0000:00:01.0", vpd_sn="SN1", parent=parent_a)
+ dev_hi = make_dev("0000:00:11.0", vpd_sn="SN2", parent=parent_b)
+ topo = NVCX_Topo({dev_lo, dev_hi})
+ assert topo.primary_pf is dev_lo
+
+
+# ---------------------------------------------------------------------------
+# NVCX_Topo — to_dict
+# ---------------------------------------------------------------------------
+
+
+class TestNVCX_Topo_ToDict:
+ def test_single_pf_returns_flat_dict(self):
+ parent = make_parent("0000:00:00.0")
+ dev = make_dev("0000:00:01.0", vpd_sn="SN123", parent=parent)
+ topo = NVCX_Topo({dev})
+ result = topo.to_dict()
+ assert result["rdma_nic_pf_bdf"] == "0000:00:01.0"
+ assert result["rdma_nic_ats"] == False
+
+ def test_single_board_multi_pf_nic(self):
+ parent = make_parent("0000:00:00.0")
+ dev1 = make_dev("0000:00:01.0", vpd_sn="SN123", parent=parent)
+ dev2 = make_dev("0000:00:02.0", vpd_sn="SN123", parent=parent)
+ topo = NVCX_Topo({dev1, dev2})
+ result = topo.to_dict()
+ boards = result["rdma_nic_boards"]
+ assert len(boards) == 1
+ board = boards[0]
+ assert board["board_sn"] == "SN123"
+ assert len(board["nics"]) == 1
+ nic_dict = board["nics"][0]
+ assert nic_dict["parent_bdf"] == "0000:00:00.0"
+ assert "ats" in nic_dict
+ assert nic_dict["pf_bdfs"] == ["0000:00:01.0", "0000:00:02.0"]
+
+ def test_multi_board_multi_pf_nics(self):
+ parent_a = make_parent("0000:00:00.0")
+ parent_b = make_parent("0000:00:10.0")
+ dev1 = make_dev("0000:00:01.0", vpd_sn="SN1", parent=parent_a)
+ dev2 = make_dev("0000:00:02.0", vpd_sn="SN1", parent=parent_a)
+ dev3 = make_dev("0000:00:11.0", vpd_sn="SN2", parent=parent_b)
+ dev4 = make_dev("0000:00:12.0", vpd_sn="SN2", parent=parent_b)
+ topo = NVCX_Topo({dev1, dev2, dev3, dev4})
+ result = topo.to_dict()
+ assert len(result["rdma_nic_boards"]) == 2
+
+ def test_single_board_multi_nic_multi_pf(self):
+ parent_a = make_parent("0000:00:00.0")
+ parent_b = make_parent("0000:00:10.0")
+ dev1 = make_dev("0000:00:01.0", vpd_sn="SN123", parent=parent_a)
+ dev2 = make_dev("0000:00:02.0", vpd_sn="SN123", parent=parent_a)
+ dev3 = make_dev("0000:00:11.0", vpd_sn="SN123", parent=parent_b)
+ dev4 = make_dev("0000:00:12.0", vpd_sn="SN123", parent=parent_b)
+ topo = NVCX_Topo({dev1, dev2, dev3, dev4})
+ result = topo.to_dict()
+ boards = result["rdma_nic_boards"]
+ assert len(boards) == 1
+ assert len(boards[0]["nics"]) == 2
+
+ def test_boards_sorted_by_sn_in_to_dict(self):
+ parent_a = make_parent("0000:00:00.0")
+ parent_b = make_parent("0000:00:10.0")
+ dev_b = make_dev("0000:00:11.0", vpd_sn="SN_B", parent=parent_b)
+ dev_a = make_dev("0000:00:01.0", vpd_sn="SN_A", parent=parent_a)
+ topo = NVCX_Topo({dev_a, dev_b})
+ boards = topo.to_dict()["rdma_nic_boards"]
+ assert boards[0]["board_sn"] == "SN_A"
+ assert boards[1]["board_sn"] == "SN_B"
+
+
+# ---------------------------------------------------------------------------
+# NVCX_Topo — topo_str_key
+# ---------------------------------------------------------------------------
+
+
+class TestNVCX_Topo_TopoStrKey:
+ def test_single_pf_returns_nic_bdf(self):
+ parent = make_parent("0000:00:00.0")
+ dev = make_dev("0000:00:01.0", vpd_sn="SN123", parent=parent)
+ topo = NVCX_Topo({dev})
+ assert topo.topo_str_key() == "RDMA NIC=0000:00:01.0"
+
+ def test_multi_pf_returns_empty(self):
+ parent = make_parent("0000:00:00.0")
+ dev1 = make_dev("0000:00:01.0", vpd_sn="SN123", parent=parent)
+ dev2 = make_dev("0000:00:02.0", vpd_sn="SN123", parent=parent)
+ topo = NVCX_Topo({dev1, dev2})
+ assert topo.topo_str_key() == ""
+
+
+# ---------------------------------------------------------------------------
+# NVCX_Topo — topo_str
+# ---------------------------------------------------------------------------
+
+
+class TestNVCX_Topo_TopoStr:
+ def test_single_pf_contains_ats(self):
+ parent = make_parent("0000:00:00.0")
+ dev = make_dev("0000:00:01.0", vpd_sn="SN123", parent=parent)
+ topo = NVCX_Topo({dev})
+ result = topo.topo_str()
+ assert "NIC ATS: no" in result
+ assert not result.endswith("\n")
+
+ def test_single_board_single_nic_multi_pf(self):
+ parent = make_parent("0000:00:00.0")
+ dev1 = make_dev("0000:00:01.0", vpd_sn="SN123", parent=parent)
+ dev2 = make_dev("0000:00:02.0", vpd_sn="SN123", parent=parent)
+ topo = NVCX_Topo({dev1, dev2})
+ result = topo.topo_str()
+ assert "NIC ATS: no" in result
+ assert "NIC PCI devices: 0000:00:01.0, 0000:00:02.0" in result
+
+ def test_single_board_multi_nic_has_nic_parent_headers(self):
+ parent_a = make_parent("0000:00:00.0")
+ parent_b = make_parent("0000:00:10.0")
+ dev1 = make_dev("0000:00:01.0", vpd_sn="SN123", parent=parent_a)
+ dev2 = make_dev("0000:00:11.0", vpd_sn="SN123", parent=parent_b)
+ topo = NVCX_Topo({dev1, dev2})
+ result = topo.topo_str()
+ assert "RDMA NIC Parent=0000:00:00.0" in result
+ assert "RDMA NIC Parent=0000:00:10.0" in result
+
+ def test_multi_board_has_nic_parent_headers(self):
+ parent_a = make_parent("0000:00:00.0")
+ parent_b = make_parent("0000:00:10.0")
+ dev1 = make_dev("0000:00:01.0", vpd_sn="SN1", parent=parent_a)
+ dev2 = make_dev("0000:00:11.0", vpd_sn="SN2", parent=parent_b)
+ topo = NVCX_Topo({dev1, dev2})
+ result = topo.topo_str()
+ assert "RDMA NIC Parent=0000:00:00.0" in result
+ assert "RDMA NIC Parent=0000:00:10.0" in result
+
+ def test_multi_board_body_indented(self):
+ parent_a = make_parent("0000:00:00.0")
+ parent_b = make_parent("0000:00:10.0")
+ dev1 = make_dev("0000:00:01.0", vpd_sn="SN1", parent=parent_a)
+ dev2 = make_dev("0000:00:11.0", vpd_sn="SN2", parent=parent_b)
+ topo = NVCX_Topo({dev1, dev2})
+ result = topo.topo_str()
+ assert "\t\t\tNIC ATS: no" in result
+
+ def test_multi_board_no_trailing_newline(self):
+ parent_a = make_parent("0000:00:00.0")
+ parent_b = make_parent("0000:00:10.0")
+ dev1 = make_dev("0000:00:01.0", vpd_sn="SN1", parent=parent_a)
+ dev2 = make_dev("0000:00:11.0", vpd_sn="SN2", parent=parent_b)
+ topo = NVCX_Topo({dev1, dev2})
+ assert not topo.topo_str().endswith("\n")
+
+
+# ---------------------------------------------------------------------------
+# NVCX_Topo — board / NIC ordering
+# ---------------------------------------------------------------------------
+
+
+class TestNVCX_Topo_Ordering:
+ def test_boards_sorted_by_sn(self):
+ parent_a = make_parent("0000:00:00.0")
+ parent_b = make_parent("0000:00:10.0")
+ dev_b = make_dev("0000:00:11.0", vpd_sn="SN_B", parent=parent_b)
+ dev_a = make_dev("0000:00:01.0", vpd_sn="SN_A", parent=parent_a)
+ topo = NVCX_Topo({dev_a, dev_b})
+ assert topo.boards[0].sn == "SN_A"
+ assert topo.boards[1].sn == "SN_B"
+
+ def test_board_nics_sorted_by_parent_bdf(self):
+ parent_lo = make_parent("0000:00:00.0")
+ parent_hi = make_parent("0000:00:10.0")
+ dev_hi = make_dev("0000:00:11.0", vpd_sn="SN", parent=parent_hi)
+ dev_lo = make_dev("0000:00:01.0", vpd_sn="SN", parent=parent_lo)
+ topo = NVCX_Topo({dev_lo, dev_hi})
+ board = topo.boards[0]
+ assert str(board.nics[0].parent.bdf) == "0000:00:00.0"
+ assert str(board.nics[1].parent.bdf) == "0000:00:10.0"
+
+ def test_board_nic_without_parent_sorts_after_nic_with_parent(self):
+ parent = make_parent("0000:00:00.0")
+ dev_parented = make_dev("0000:00:01.0", vpd_sn="SN", parent=parent)
+ dev_orphan = make_dev("0000:00:02.0", vpd_sn="SN", parent=None)
+ nic_parented = NVCX_Topo.NIC({dev_parented})
+ nic_orphan = NVCX_Topo.NIC({dev_orphan})
+ board = NVCX_Topo.Board({nic_parented, nic_orphan})
+ assert board.nics[0] is nic_parented
+ assert board.nics[1] is nic_orphan
diff --git a/kernel-headers/rdma/mana-abi.h b/kernel-headers/rdma/mana-abi.h
index a75bf32..32cbbfc 100644
--- a/kernel-headers/rdma/mana-abi.h
+++ b/kernel-headers/rdma/mana-abi.h
@@ -25,7 +25,7 @@
struct mana_ib_create_cq {
__aligned_u64 buf_addr;
- __u16 flags;
+ __u16 comp_mask;
__u16 reserved0;
__u32 reserved1;
};
@@ -57,6 +57,17 @@
__u32 queue_id[4];
};
+struct mana_ib_create_uc_qp {
+ __aligned_u64 queue_buf[3];
+ __u32 queue_size[3];
+ __u32 comp_mask;
+};
+
+struct mana_ib_create_uc_qp_resp {
+ __u32 queue_id[3];
+ __u32 reserved;
+};
+
struct mana_ib_create_wq {
__aligned_u64 wq_buf_addr;
__u32 wq_buf_size;
@@ -87,4 +98,26 @@
struct rss_resp_entry entries[64];
};
+enum mana_ib_ucontext_support {
+ MANA_IB_UCNTX_ALLOC_PDN_SUPPORT = 1 << 0,
+};
+
+struct mana_ib_alloc_ucontext_resp {
+ __aligned_u64 comp_mask;
+};
+
+enum mana_ib_create_pd_flags {
+ MANA_IB_PD_SHORT_PDN = 1 << 0,
+};
+
+struct mana_ib_alloc_pd {
+ __u32 comp_mask;
+ __u32 reserved;
+};
+
+struct mana_ib_alloc_pd_resp {
+ __u32 pdn;
+ __u32 reserved;
+};
+
#endif
diff --git a/providers/mana/CMakeLists.txt b/providers/mana/CMakeLists.txt
index 05011be..3ae2b02 100644
--- a/providers/mana/CMakeLists.txt
+++ b/providers/mana/CMakeLists.txt
@@ -1,5 +1,5 @@
rdma_shared_provider(mana libmana.map
- 1 1.0.${PACKAGE_VERSION}
+ 1 1.1.${PACKAGE_VERSION}
mana.c
manadv.c
qp.c
diff --git a/providers/mana/cq.c b/providers/mana/cq.c
index 367d5a2..09da80c 100644
--- a/providers/mana/cq.c
+++ b/providers/mana/cq.c
@@ -165,7 +165,7 @@
cmd_drv = &cmd.drv_payload;
cmd_drv->buf_addr = (uintptr_t)cq->buf;
- cmd_drv->flags = flags;
+ cmd_drv->comp_mask = flags;
resp.cqid = UINT32_MAX;
ret = ibv_cmd_create_cq(context, cq->cqe, channel, comp_vector,
@@ -284,6 +284,25 @@
return produced;
}
+static inline int handle_requester_cqe(struct mana_qp *qp, struct gdma_cqe *cqe, struct ibv_wc *wc)
+{
+ struct mana_gdma_queue *send_queue = mana_ib_get_sreq(qp);
+ struct shadow_wqe_header *wqe;
+ int produced = 0;
+
+ while (!produced && (wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq)) != NULL) {
+ send_queue->cons_idx += wqe->posted_wqe_size_in_bu;
+ send_queue->cons_idx &= GDMA_QUEUE_OFFSET_MASK;
+ if (wqe->flags != MANA_NO_SIGNAL_WC) {
+ fill_verbs_from_shadow_wqe(qp, wc, wqe);
+ produced++;
+ }
+ shadow_queue_advance_consumer(&qp->shadow_sq);
+ }
+
+ return produced;
+}
+
static inline int handle_rc_requester_cqe(struct mana_qp *qp, struct gdma_cqe *cqe,
struct ibv_wc *wc, int nwc, bool *consumed)
{
@@ -443,6 +462,8 @@
return handle_error_cqe(qp, cqe, wc, nwc, consumed);
else if (cqe->rdma_cqe.cqe_type == CQE_TYPE_ARMED_CMPL)
return handle_rc_requester_cqe(qp, cqe, wc, nwc, consumed);
+ else if (cqe->is_sq && cqe->rdma_cqe.cqe_type == CQE_TYPE_UD_SEND)
+ return handle_requester_cqe(qp, cqe, wc);
else
return handle_responder_cqe(qp, cqe, wc);
}
diff --git a/providers/mana/doorbells.h b/providers/mana/doorbells.h
index d805cdb..7651e76 100644
--- a/providers/mana/doorbells.h
+++ b/providers/mana/doorbells.h
@@ -87,6 +87,16 @@
mmio_flush_writes();
}
+/* Has HW already produced the CQE at this index? Owner bits are written by HW,
+ * so a match proves idx < cur_cqe. Only meaningful for idx >= cq->head.
+ */
+static inline bool gdma_cq_idx_produced(struct mana_cq *cq, uint32_t idx)
+{
+ struct gdma_cqe *cqe = ((struct gdma_cqe *)cq->buf) + (idx % cq->cqe);
+
+ return cqe->owner_bits == ((idx / cq->cqe) & CQ_OWNER_MASK);
+}
+
static inline void gdma_ring_cq_doorbell(struct mana_cq *cq, uint8_t arm)
{
union gdma_doorbell_entry e;
@@ -94,8 +104,14 @@
uint32_t max_credit = cq->cqe << (GDMA_CQE_OWNER_BITS - 1);
if (cq->poll_credit >= max_credit) {
- // To address the use-case of ibv that re-arms the CQ without polling
- cq->poll_credit++;
+ // To address the use-case of ibv that re-arms the CQ without polling.
+ // (prod_idx + poll_credit - max_credit) is the index last given to HW,
+ // so only claim the next one once HW has produced it. Nothing produced
+ // implies no notification fired, so the CQ is still armed.
+ if (gdma_cq_idx_produced(cq, prod_idx + cq->poll_credit - max_credit))
+ cq->poll_credit++;
+ else
+ return;
} else {
// Set index of already polled CQE for unarm
cq->poll_credit = max_credit - (arm ? 0 : 1);
diff --git a/providers/mana/libmana.map b/providers/mana/libmana.map
index ab66295..24af016 100644
--- a/providers/mana/libmana.map
+++ b/providers/mana/libmana.map
@@ -6,3 +6,8 @@
manadv_init_obj;
local: *;
};
+
+MANA_1.1 {
+ global:
+ manadv_alloc_pd;
+} MANA_1.0;
diff --git a/providers/mana/man/CMakeLists.txt b/providers/mana/man/CMakeLists.txt
index 24f1859..3458443 100644
--- a/providers/mana/man/CMakeLists.txt
+++ b/providers/mana/man/CMakeLists.txt
@@ -2,4 +2,5 @@
manadv.7.md
manadv_init_obj.3.md
manadv_set_context_attr.3.md
+ manadv_alloc_pd.3.md
)
diff --git a/providers/mana/man/manadv_alloc_pd.3.md b/providers/mana/man/manadv_alloc_pd.3.md
new file mode 100644
index 0000000..b121ea2
--- /dev/null
+++ b/providers/mana/man/manadv_alloc_pd.3.md
@@ -0,0 +1,36 @@
+---
+layout: page
+title: manadv_alloc_pd
+section: 3
+tagline: Verbs
+---
+
+# NAME
+manadv_alloc_pd \- Create a MANA specific PD for the RDMA device context.
+
+# SYNOPSIS"
+```c
+#include <infiniband/manadv.h>
+
+struct ibv_pd *manadv_alloc_pd(struct ibv_context *context, uint32_t flags);
+```
+
+# DESCRIPTION
+**manadv_alloc_pd()** allocates a PD for the RDMA device context with additional
+creation flags.
+
+# ARGUMENTS
+*context*
+: RDMA device context to work on.
+
+*flags*
+: A bitwise OR of the various values described below.
+
+ MANADV_PD_FLAGS_SHORT_PDN:
+ allocates a PD with 16 bit PDN.
+
+# RETURN VALUE
+returns a pointer to the allocated PD, or NULL if the request fails.
+
+# AUTHORS
+Konstantin Taranov <kotaranov@microsoft.com>
diff --git a/providers/mana/man/manadv_init_obj.3.md b/providers/mana/man/manadv_init_obj.3.md
index 575ea34..a4dd712 100644
--- a/providers/mana/man/manadv_init_obj.3.md
+++ b/providers/mana/man/manadv_init_obj.3.md
@@ -48,6 +48,10 @@
void *db_page;
};
+struct manadv_pd {
+ uint32_t pdn;
+};
+
struct manadv_obj {
struct {
struct ibv_qp *in;
@@ -63,6 +67,11 @@
struct ibv_wq *in;
struct manadv_rwq *out;
} rwq;
+
+ struct {
+ struct ibv_pd *in;
+ struct manadv_pd *out;
+ } pd;
};
```
@@ -74,6 +83,7 @@
MANADV_OBJ_QP = 1 << 0,
MANADV_OBJ_CQ = 1 << 1,
MANADV_OBJ_RWQ = 1 << 2,
+ MANADV_OBJ_PD = 1 << 3,
};
```
# RETURN VALUE
diff --git a/providers/mana/mana.c b/providers/mana/mana.c
index a59248b..87411d4 100644
--- a/providers/mana/mana.c
+++ b/providers/mana/mana.c
@@ -21,9 +21,10 @@
#include "mana.h"
DECLARE_DRV_CMD(mana_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT, empty,
- empty);
+ mana_ib_alloc_ucontext_resp);
-DECLARE_DRV_CMD(mana_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, empty, empty);
+DECLARE_DRV_CMD(mana_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, mana_ib_alloc_pd,
+ mana_ib_alloc_pd_resp);
static const struct verbs_match_ent hca_table[] = {
VERBS_DRIVER_ID(RDMA_DRIVER_MANA),
@@ -114,19 +115,34 @@
return ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd));
}
-struct ibv_pd *mana_alloc_pd(struct ibv_context *context)
+struct ibv_pd *mana_alloc_pd_ex(struct ibv_context *context, uint32_t flags)
{
- struct ibv_alloc_pd cmd;
- struct mana_alloc_pd_resp resp;
+ struct mana_context *mctx = to_mctx(context);
+ struct mana_alloc_pd cmd = {};
+ struct mana_ib_alloc_pd *cmd_drv = &cmd.drv_payload;
+ struct mana_alloc_pd_resp resp = {};
+ size_t cmd_size = sizeof(cmd.ibv_cmd); /* v0 size */
struct mana_pd *pd;
int ret;
+ if ((flags & MANADV_PD_FLAGS_SHORT_PDN) &&
+ !(mctx->comp_mask & MANA_IB_UCNTX_ALLOC_PDN_SUPPORT)) {
+ errno = EOPNOTSUPP;
+ return NULL;
+ }
+
pd = calloc(1, sizeof(*pd));
if (!pd)
return NULL;
- ret = ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof(cmd),
- &resp.ibv_resp, sizeof(resp));
+ if (mctx->comp_mask & MANA_IB_UCNTX_ALLOC_PDN_SUPPORT)
+ cmd_size = sizeof(cmd); /* v1 size */
+
+ if (flags & MANADV_PD_FLAGS_SHORT_PDN)
+ cmd_drv->comp_mask |= MANA_IB_PD_SHORT_PDN;
+
+ ret = ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd.ibv_cmd, cmd_size,
+ &resp.ibv_resp, sizeof(resp));
if (ret) {
verbs_err(verbs_get_ctx(context), "Failed to allocate PD\n");
errno = ret;
@@ -134,9 +150,16 @@
return NULL;
}
+ pd->pdn = resp.pdn;
+
return &pd->ibv_pd;
}
+static struct ibv_pd *mana_alloc_pd(struct ibv_context *context)
+{
+ return mana_alloc_pd_ex(context, 0);
+}
+
struct ibv_pd *
mana_alloc_parent_domain(struct ibv_context *context,
struct ibv_parent_domain_init_attr *attr)
@@ -452,7 +475,7 @@
{
int ret, i;
struct mana_context *context;
- struct mana_alloc_ucontext_resp resp;
+ struct mana_alloc_ucontext_resp resp = {};
struct ibv_get_context cmd;
context = verbs_init_and_alloc_context(ibdev, cmd_fd, context, ibv_ctx,
@@ -468,6 +491,8 @@
goto free_ctx;
}
+ context->comp_mask = resp.drv_payload.comp_mask;
+
verbs_set_ops(&context->ibv_ctx, &mana_ctx_ops);
pthread_mutex_init(&context->qp_table_mutex, NULL);
diff --git a/providers/mana/mana.h b/providers/mana/mana.h
index 1f8a147..98318d1 100644
--- a/providers/mana/mana.h
+++ b/providers/mana/mana.h
@@ -49,7 +49,8 @@
USER_RNIC_SEND_QUEUE_RESPONDER = 1,
USER_RNIC_RECV_QUEUE_REQUESTER = 2,
USER_RNIC_RECV_QUEUE_RESPONDER = 3,
- USER_RNIC_QUEUE_TYPE_MAX = 4,
+ USER_RNIC_SEND_QUEUE_MM = 4,
+ USER_RNIC_QUEUE_TYPE_MAX = 5,
};
#define QUEUE_TYPE_MASK 0x3
@@ -89,6 +90,7 @@
struct mana_table qp_rtable[MANA_QP_TABLE_SIZE];
struct mana_table qp_stable[MANA_QP_TABLE_SIZE];
pthread_mutex_t qp_table_mutex;
+ uint64_t comp_mask;
struct manadv_ctx_allocators extern_alloc;
void *db_page;
@@ -196,6 +198,7 @@
struct mana_pd {
struct ibv_pd ibv_pd;
+ uint32_t pdn;
struct mana_pd *mprotection_domain;
};
@@ -216,7 +219,7 @@
int mana_query_port(struct ibv_context *context, uint8_t port,
struct ibv_port_attr *attr);
-struct ibv_pd *mana_alloc_pd(struct ibv_context *context);
+struct ibv_pd *mana_alloc_pd_ex(struct ibv_context *context, uint32_t flags);
struct ibv_pd *
mana_alloc_parent_domain(struct ibv_context *context,
struct ibv_parent_domain_init_attr *attr);
diff --git a/providers/mana/manadv.c b/providers/mana/manadv.c
index 4b40d05..2392adf 100644
--- a/providers/mana/manadv.c
+++ b/providers/mana/manadv.c
@@ -42,7 +42,7 @@
int manadv_init_obj(struct manadv_obj *obj, uint64_t obj_type)
{
- if (obj_type & ~(MANADV_OBJ_QP | MANADV_OBJ_CQ | MANADV_OBJ_RWQ))
+ if (obj_type & ~(MANADV_OBJ_QP | MANADV_OBJ_CQ | MANADV_OBJ_RWQ | MANADV_OBJ_PD))
return EINVAL;
if (obj_type & MANADV_OBJ_QP) {
@@ -84,5 +84,17 @@
obj->rwq.out->db_page = ctx->db_page;
}
+ if (obj_type & MANADV_OBJ_PD) {
+ struct ibv_pd *ibpd = obj->pd.in;
+ struct mana_pd *pd = container_of(ibpd, struct mana_pd, ibv_pd);
+
+ obj->pd.out->pdn = pd->pdn;
+ }
+
return 0;
}
+
+struct ibv_pd *manadv_alloc_pd(struct ibv_context *context, uint32_t flags)
+{
+ return mana_alloc_pd_ex(context, flags);
+}
diff --git a/providers/mana/manadv.h b/providers/mana/manadv.h
index 27c8fe9..87f6e8f 100644
--- a/providers/mana/manadv.h
+++ b/providers/mana/manadv.h
@@ -52,6 +52,10 @@
void *db_page;
};
+struct manadv_pd {
+ uint32_t pdn;
+};
+
struct manadv_obj {
struct {
struct ibv_qp *in;
@@ -67,16 +71,27 @@
struct ibv_wq *in;
struct manadv_rwq *out;
} rwq;
+ struct {
+ struct ibv_pd *in;
+ struct manadv_pd *out;
+ } pd;
};
enum manadv_obj_type {
MANADV_OBJ_QP = 1 << 0,
MANADV_OBJ_CQ = 1 << 1,
MANADV_OBJ_RWQ = 1 << 2,
+ MANADV_OBJ_PD = 1 << 3,
+};
+
+enum {
+ MANADV_PD_FLAGS_SHORT_PDN = 1 << 0,
};
int manadv_init_obj(struct manadv_obj *obj, uint64_t obj_type);
+struct ibv_pd *manadv_alloc_pd(struct ibv_context *context, uint32_t flags);
+
#ifdef __cplusplus
}
#endif
diff --git a/providers/mana/qp.c b/providers/mana/qp.c
index b6a9a7e..752959b 100644
--- a/providers/mana/qp.c
+++ b/providers/mana/qp.c
@@ -31,6 +31,9 @@
DECLARE_DRV_CMD(mana_create_rc_qp, IB_USER_VERBS_CMD_CREATE_QP,
mana_ib_create_rc_qp, mana_ib_create_rc_qp_resp);
+DECLARE_DRV_CMD(mana_create_uc_qp, IB_USER_VERBS_CMD_CREATE_QP,
+ mana_ib_create_uc_qp, mana_ib_create_uc_qp_resp);
+
static struct ibv_qp *mana_create_qp_raw(struct ibv_pd *ibpd,
struct ibv_qp_init_attr *attr)
{
@@ -216,46 +219,142 @@
uint32_t size = 0;
uint32_t sges = 0;
- if (attr->qp_type == IBV_QPT_RC) {
- switch (type) {
- case USER_RNIC_SEND_QUEUE_REQUESTER:
- /* WQE must have at least one SGE */
- /* For write with imm we need one extra SGE */
- sges = max(1U, attr->cap.max_send_sge) + 1;
- size = attr->cap.max_send_wr * get_large_wqe_size(sges);
- break;
- case USER_RNIC_SEND_QUEUE_RESPONDER:
- size = MANA_PAGE_SIZE;
- break;
- case USER_RNIC_RECV_QUEUE_REQUESTER:
- size = MANA_PAGE_SIZE;
- break;
- case USER_RNIC_RECV_QUEUE_RESPONDER:
- /* WQE must have at least one SGE */
- sges = max(1U, attr->cap.max_recv_sge);
- size = attr->cap.max_recv_wr * get_wqe_size(sges);
- break;
- default:
- return 0;
- }
+ switch (type) {
+ case USER_RNIC_SEND_QUEUE_REQUESTER:
+ /* WQE must have at least one SGE */
+ /* For write with imm we need one extra SGE */
+ sges = max(1U, attr->cap.max_send_sge) + 1;
+ size = align_hw_size(attr->cap.max_send_wr * get_large_wqe_size(sges));
+ break;
+ case USER_RNIC_SEND_QUEUE_RESPONDER:
+ if (attr->qp_type == IBV_QPT_RC)
+ size = align_hw_size(MANA_PAGE_SIZE);
+ break;
+ case USER_RNIC_RECV_QUEUE_REQUESTER:
+ if (attr->qp_type == IBV_QPT_RC)
+ size = align_hw_size(MANA_PAGE_SIZE);
+ break;
+ case USER_RNIC_RECV_QUEUE_RESPONDER:
+ /* WQE must have at least one SGE */
+ sges = max(1U, attr->cap.max_recv_sge);
+ size = align_hw_size(attr->cap.max_recv_wr * get_wqe_size(sges));
+ break;
+ case USER_RNIC_SEND_QUEUE_MM:
+ sges = 2;
+ size = align_hw_size(attr->cap.max_send_wr * get_large_wqe_size(sges));
+ break;
+ default:
+ return 0;
}
- size = align_hw_size(size);
-
if (attr->qp_type == IBV_QPT_RC && type == USER_RNIC_SEND_QUEUE_REQUESTER)
size += sizeof(struct mana_ib_rollback_shared_mem);
return size;
}
-static struct ibv_qp *mana_create_qp_rnic(struct ibv_pd *ibpd,
- struct ibv_qp_init_attr *attr)
+static int mana_create_cmd_qp_rc(struct mana_qp *qp, struct ibv_pd *ibpd,
+ struct ibv_qp_init_attr *attr)
{
- struct mana_context *ctx = to_mctx(ibpd->context);
struct mana_ib_create_rc_qp_resp *qp_resp_drv;
struct mana_create_rc_qp_resp qp_resp = {};
struct mana_ib_create_rc_qp *qp_cmd_drv;
struct mana_create_rc_qp qp_cmd = {};
+ int ret, i;
+
+ qp_cmd_drv = &qp_cmd.drv_payload;
+ qp_resp_drv = &qp_resp.drv_payload;
+
+ for (i = 0; i < USER_RNIC_QUEUE_TYPE_MAX; ++i) {
+ if (i == USER_RNIC_SEND_QUEUE_MM)
+ continue;
+ qp_cmd_drv->queue_buf[i] = (uintptr_t)qp->rnic_qp.queues[i].buffer;
+ qp_cmd_drv->queue_size[i] = qp->rnic_qp.queues[i].size;
+ }
+
+ ret = ibv_cmd_create_qp(ibpd, &qp->ibqp.qp, attr, &qp_cmd.ibv_cmd,
+ sizeof(qp_cmd), &qp_resp.ibv_resp,
+ sizeof(qp_resp));
+ if (ret) {
+ verbs_err(verbs_get_ctx(ibpd->context), "Create QP failed\n");
+ return ret;
+ }
+
+ for (i = 0; i < USER_RNIC_QUEUE_TYPE_MAX; ++i) {
+ if (i == USER_RNIC_SEND_QUEUE_MM)
+ continue;
+ qp->rnic_qp.queues[i].id = qp_resp_drv->queue_id[i];
+ }
+
+ return 0;
+}
+
+enum {
+ MANA_UC_UDATA_SQR = 0,
+ MANA_UC_UDATA_RQR = 1,
+ MANA_UC_UDATA_SMQ = 2,
+};
+
+static int mana_create_cmd_qp_uc(struct mana_qp *qp, struct ibv_pd *ibpd,
+ struct ibv_qp_init_attr *attr)
+{
+ struct mana_ib_create_uc_qp_resp *qp_resp_drv;
+ struct mana_create_uc_qp_resp qp_resp = {};
+ struct mana_ib_create_uc_qp *qp_cmd_drv;
+ struct mana_create_uc_qp qp_cmd = {};
+ int ret;
+
+ qp_cmd_drv = &qp_cmd.drv_payload;
+ qp_resp_drv = &qp_resp.drv_payload;
+
+ qp_cmd_drv->queue_buf[MANA_UC_UDATA_SQR] =
+ (uintptr_t)qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER].buffer;
+ qp_cmd_drv->queue_size[MANA_UC_UDATA_SQR] =
+ qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER].size;
+
+ qp_cmd_drv->queue_buf[MANA_UC_UDATA_RQR] =
+ (uintptr_t)qp->rnic_qp.queues[USER_RNIC_RECV_QUEUE_RESPONDER].buffer;
+ qp_cmd_drv->queue_size[MANA_UC_UDATA_RQR] =
+ qp->rnic_qp.queues[USER_RNIC_RECV_QUEUE_RESPONDER].size;
+
+ qp_cmd_drv->queue_buf[MANA_UC_UDATA_SMQ] =
+ (uintptr_t)qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].buffer;
+ qp_cmd_drv->queue_size[MANA_UC_UDATA_SMQ] =
+ qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].size;
+
+ ret = ibv_cmd_create_qp(ibpd, &qp->ibqp.qp, attr, &qp_cmd.ibv_cmd,
+ sizeof(qp_cmd), &qp_resp.ibv_resp,
+ sizeof(qp_resp));
+ if (ret) {
+ verbs_err(verbs_get_ctx(ibpd->context), "Create QP failed\n");
+ return ret;
+ }
+
+ qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER].id =
+ qp_resp_drv->queue_id[MANA_UC_UDATA_SQR];
+ qp->rnic_qp.queues[USER_RNIC_RECV_QUEUE_RESPONDER].id =
+ qp_resp_drv->queue_id[MANA_UC_UDATA_RQR];
+ qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].id =
+ qp_resp_drv->queue_id[MANA_UC_UDATA_SMQ];
+
+ return 0;
+}
+
+static int mana_create_cmd_qp(struct mana_qp *qp, struct ibv_pd *ibpd,
+ struct ibv_qp_init_attr *attr)
+{
+ if (attr->qp_type == IBV_QPT_RC)
+ return mana_create_cmd_qp_rc(qp, ibpd, attr);
+ else if (attr->qp_type == IBV_QPT_UC)
+ return mana_create_cmd_qp_uc(qp, ibpd, attr);
+ else
+ return -EOPNOTSUPP;
+}
+
+static struct ibv_qp *mana_create_qp_rnic(struct ibv_pd *ibpd,
+ struct ibv_qp_init_attr *attr)
+{
+ struct mana_context *ctx = to_mctx(ibpd->context);
struct mana_qp *qp;
int ret, i;
@@ -263,9 +362,6 @@
if (!qp)
return NULL;
- qp_cmd_drv = &qp_cmd.drv_payload;
- qp_resp_drv = &qp_resp.drv_payload;
-
pthread_spin_init(&qp->sq_lock, PTHREAD_PROCESS_PRIVATE);
pthread_spin_init(&qp->rq_lock, PTHREAD_PROCESS_PRIVATE);
qp->sq_sig_all = attr->sq_sig_all;
@@ -291,29 +387,18 @@
if (qp->rnic_qp.queues[i].size != 0 && !qp->rnic_qp.queues[i].buffer) {
verbs_err(verbs_get_ctx(ibpd->context),
- "Failed to allocate memory for RC queue %d\n", i);
+ "Failed to allocate memory for queue %d\n", i);
errno = ENOMEM;
goto destroy_queues;
}
-
- qp_cmd_drv->queue_buf[i] = (uintptr_t)qp->rnic_qp.queues[i].buffer;
- qp_cmd_drv->queue_size[i] = qp->rnic_qp.queues[i].size;
}
- mana_ib_init_rb_shmem(qp);
-
- ret = ibv_cmd_create_qp(ibpd, &qp->ibqp.qp, attr, &qp_cmd.ibv_cmd,
- sizeof(qp_cmd), &qp_resp.ibv_resp,
- sizeof(qp_resp));
+ ret = mana_create_cmd_qp(qp, ibpd, attr);
if (ret) {
- verbs_err(verbs_get_ctx(ibpd->context), "Create QP failed\n");
errno = ret;
- goto free_rb;
+ goto destroy_queues;
}
- for (i = 0; i < USER_RNIC_QUEUE_TYPE_MAX; ++i)
- qp->rnic_qp.queues[i].id = qp_resp_drv->queue_id[i];
-
qp->ibqp.qp.qp_num = qp->rnic_qp.queues[USER_RNIC_RECV_QUEUE_RESPONDER].id;
ret = mana_store_qp(ctx, qp);
@@ -322,12 +407,12 @@
goto destroy_qp;
}
+ mana_ib_init_rb_shmem(qp);
+
return &qp->ibqp.qp;
destroy_qp:
ibv_cmd_destroy_qp(&qp->ibqp.qp);
-free_rb:
- mana_ib_deinit_rb_shmem(qp);
destroy_queues:
while (i-- > 0)
mana_dealloc_mem(qp->rnic_qp.queues[i].buffer, qp->rnic_qp.queues[i].size);
@@ -346,6 +431,7 @@
case IBV_QPT_RAW_PACKET:
return mana_create_qp_raw(ibpd, attr);
case IBV_QPT_RC:
+ case IBV_QPT_UC:
return mana_create_qp_rnic(ibpd, attr);
default:
verbs_err(verbs_get_ctx(ibpd->context),
@@ -382,7 +468,8 @@
if (attr_mask & IBV_QP_SQ_PSN) {
qp->sq_ssn = 1;
qp->sq_psn = attr->sq_psn;
- gdma_arm_normal_cqe(mana_ib_get_rreq(qp), attr->sq_psn);
+ if (qp->ibqp.qp.qp_type == IBV_QPT_RC)
+ gdma_arm_normal_cqe(mana_ib_get_rreq(qp), attr->sq_psn);
}
break;
default:
@@ -397,7 +484,7 @@
struct ibv_modify_qp cmd = {};
int err;
- if (ibqp->qp_type != IBV_QPT_RC)
+ if (ibqp->qp_type != IBV_QPT_RC && ibqp->qp_type != IBV_QPT_UC)
return EOPNOTSUPP;
pthread_spin_lock(&qp->sq_lock);
@@ -443,7 +530,7 @@
struct mana_context *ctx = to_mctx(ibqp->context);
int ret, i;
- if (ibqp->qp_type == IBV_QPT_RC) {
+ if (ibqp->qp_type == IBV_QPT_RC || ibqp->qp_type == IBV_QPT_UC) {
mana_remove_qp(ctx, qp);
mana_drain_cqes(qp);
}
@@ -459,6 +546,7 @@
ctx->extern_alloc.free(qp->raw_qp.send_buf, ctx->extern_alloc.data);
break;
case IBV_QPT_RC:
+ case IBV_QPT_UC:
pthread_spin_destroy(&qp->sq_lock);
pthread_spin_destroy(&qp->rq_lock);
destroy_shadow_queue(&qp->shadow_sq);
diff --git a/providers/mana/rollback.h b/providers/mana/rollback.h
index 59621a5..c908f3f 100644
--- a/providers/mana/rollback.h
+++ b/providers/mana/rollback.h
@@ -39,6 +39,8 @@
static inline void mana_ib_init_rb_shmem(struct mana_qp *qp)
{
+ if (qp->ibqp.qp.qp_type != IBV_QPT_RC)
+ return;
// take some bytes for rollback memory
struct mana_gdma_queue *req_sq =
&qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER];
@@ -54,6 +56,8 @@
static inline void mana_ib_deinit_rb_shmem(struct mana_qp *qp)
{
+ if (qp->ibqp.qp.qp_type != IBV_QPT_RC)
+ return;
// return back bytes for rollback memory
struct mana_gdma_queue *req_sq =
&qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER];
@@ -62,6 +66,9 @@
static inline void mana_ib_reset_rb_shmem(struct mana_qp *qp)
{
+ if (qp->ibqp.qp.qp_type != IBV_QPT_RC)
+ return;
+
struct mana_ib_rollback_shared_mem *rb_shmem =
mana_ib_get_rollback_sh_mem(qp);
@@ -71,6 +78,9 @@
static inline void mana_ib_update_shared_mem_right_offset(struct mana_qp *qp, uint32_t offset_in_bu)
{
+ if (qp->ibqp.qp.qp_type != IBV_QPT_RC)
+ return;
+
struct mana_ib_rollback_shared_mem *rb_shmem =
mana_ib_get_rollback_sh_mem(qp);
diff --git a/providers/mana/wr.c b/providers/mana/wr.c
index 88ee8ad..e4d7c38 100644
--- a/providers/mana/wr.c
+++ b/providers/mana/wr.c
@@ -191,6 +191,7 @@
{
switch (ibqp->qp_type) {
case IBV_QPT_RC:
+ case IBV_QPT_UC:
return mana_ib_post_recv(ibqp, wr, bad);
default:
verbs_err(verbs_get_ctx(ibqp->context), "QPT not supported %d\n", ibqp->qp_type);
@@ -350,7 +351,7 @@
&send_oob, oob_sge, num_sge, MTU_SIZE(qp->mtu), flags, &gdma_wqe);
if (ret) {
verbs_err(verbs_get_ctx(qp->ibqp.qp.context),
- "rc post send error, ret %d\n", ret);
+ "post send error, ret %d\n", ret);
goto cleanup;
}
@@ -430,6 +431,7 @@
{
switch (ibqp->qp_type) {
case IBV_QPT_RC:
+ case IBV_QPT_UC:
return mana_ib_post_send(ibqp, wr, bad);
default:
verbs_err(verbs_get_ctx(ibqp->context), "QPT not supported %d\n", ibqp->qp_type);
diff --git a/providers/mlx4/cq.c b/providers/mlx4/cq.c
index 61313b8..e17ae22 100644
--- a/providers/mlx4/cq.c
+++ b/providers/mlx4/cq.c
@@ -700,7 +700,8 @@
* Now sweep backwards through the CQ, removing CQ entries
* that match our QP by copying older entries on top of them.
*/
- while ((int) --prod_index - (int) cq->cons_index >= 0) {
+ while (prod_index != cq->cons_index) {
+ --prod_index;
cqe = get_cqe(cq, prod_index & cq->verbs_cq.cq.cqe);
cqe += cqe_inc;
if (srq && srq->ext_srq &&
diff --git a/providers/mlx5/cq.c b/providers/mlx5/cq.c
index eeaf4e6..f892658 100644
--- a/providers/mlx5/cq.c
+++ b/providers/mlx5/cq.c
@@ -1842,7 +1842,8 @@
* that match our QP by copying older entries on top of them.
*/
cqe_version = (to_mctx(cq->verbs_cq.cq.context))->cqe_version;
- while ((int) --prod_index - (int) cq->cons_index >= 0) {
+ while (prod_index != cq->cons_index) {
+ --prod_index;
cqe = get_cqe(cq, prod_index & cq->verbs_cq.cq.cqe);
cqe64 = (cq->cqe_sz == 64) ? cqe : cqe + 64;
if (free_res_cqe(cqe64, rsn, srq, cqe_version)) {
diff --git a/providers/mlx5/dbrec.c b/providers/mlx5/dbrec.c
index c5d1c3f..ae99cd9 100644
--- a/providers/mlx5/dbrec.c
+++ b/providers/mlx5/dbrec.c
@@ -105,8 +105,10 @@
mparent_domain->pd_context, 8, 8,
MLX5DV_RES_TYPE_DBR);
- if (db == IBV_ALLOCATOR_USE_DEFAULT)
+ if (db == IBV_ALLOCATOR_USE_DEFAULT) {
+ db = NULL;
goto default_alloc;
+ }
if (!db)
return NULL;
diff --git a/providers/mthca/cq.c b/providers/mthca/cq.c
index dd8baca..3fff17f 100644
--- a/providers/mthca/cq.c
+++ b/providers/mthca/cq.c
@@ -563,7 +563,8 @@
* Now sweep backwards through the CQ, removing CQ entries
* that match our QP by copying older entries on top of them.
*/
- while ((int) --prod_index - (int) cq->cons_index >= 0) {
+ while (prod_index != cq->cons_index) {
+ --prod_index;
cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe);
if (cqe->my_qpn == htobe32(qpn)) {
if (srq && is_recv_cqe(cqe))
diff --git a/pyverbs/__init__.py b/pyverbs/__init__.py
index e69de29..9226fe7 100644
--- a/pyverbs/__init__.py
+++ b/pyverbs/__init__.py
@@ -0,0 +1 @@
+from .__version__ import __version__
diff --git a/pyverbs/__version__.py.in b/pyverbs/__version__.py.in
new file mode 100644
index 0000000..280beb2
--- /dev/null
+++ b/pyverbs/__version__.py.in
@@ -0,0 +1 @@
+__version__ = "@PACKAGE_VERSION@"
