| # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) |
| # Copyright (c) 2019 Mellanox Technologies, Inc . All rights reserved. See COPYING file |
| |
| import abc |
| |
| from pyverbs.cmid import CMID, AddrInfo, CMEventChannel, ConnParam, UDParam |
| from pyverbs.qp import QPCap, QPInitAttr, QPAttr, QP |
| from pyverbs.pyverbs_error import PyverbsUserError |
| import pyverbs.cm_enums as ce |
| import pyverbs.enums as e |
| from pyverbs.cq import CQ |
| |
| |
| GRH_SIZE = 40 |
| qp_type_per_ps = {ce.RDMA_PS_TCP: e.IBV_QPT_RC, ce.RDMA_PS_UDP: e.IBV_QPT_UD, |
| ce.RDMA_PS_IPOIB : e.IBV_QPT_UD} |
| |
| |
| class CMResources(abc.ABC): |
| """ |
| CMResources class is an abstract base class which contains basic resources |
| for RDMA CM communication. |
| """ |
| def __init__(self, addr=None, passive=None, **kwargs): |
| """ |
| :param addr: Local address to bind to. |
| :param passive: Indicate if this CM is the passive CM. |
| :param kwargs: Arguments: |
| * *port* (str) |
| Port number of the address |
| * *with_ext_qp* (bool) |
| If set, an external RC QP will be created and used by RDMACM |
| * *port_space* (str) |
| If set, indicates the CMIDs port space |
| """ |
| self.qp_init_attr = None |
| self.passive = passive |
| self.with_ext_qp = kwargs.get('with_ext_qp', False) |
| self.port = kwargs.get('port') if kwargs.get('port') else '7471' |
| self.ib_port = int(kwargs.get('ib_port', '1')) |
| self.port_space = kwargs.get('port_space', ce.RDMA_PS_TCP) |
| self.remote_operation = kwargs.get('remote_op') |
| self.qp_type = qp_type_per_ps[self.port_space] |
| self.qp_init_attr = QPInitAttr(qp_type=self.qp_type, cap=QPCap()) |
| self.connected = False |
| # When passive side (server) listens to incoming connection requests, |
| # for each new request it creates a new cmid which is used to establish |
| # the connection with the remote side |
| self.msg_size = 1024 |
| self.num_msgs = 10 |
| self.channel = None |
| self.cq = None |
| self.qps = {} |
| self.mr = None |
| self.remote_qpn = None |
| self.ud_params = None |
| self.child_ids = {} |
| self.cmids = {} |
| if self.passive: |
| self.ai = AddrInfo(src=addr, src_service=self.port, |
| port_space=self.port_space, flags=ce.RAI_PASSIVE) |
| else: |
| self.ai = AddrInfo(src=addr, dst=addr, dst_service=self.port, |
| port_space=self.port_space) |
| |
| @property |
| def child_id(self): |
| if self.child_ids: |
| return self.child_ids[0] |
| |
| @property |
| def cmid(self): |
| if self.cmids: |
| return self.cmids[0] |
| |
| @property |
| def qp(self): |
| if self.qps: |
| return self.qps[0] |
| |
| def create_mr(self): |
| cmid = self.child_id if self.passive else self.cmid |
| mr_remote_function = {None: cmid.reg_msgs, 'read': cmid.reg_read, |
| 'write': cmid.reg_write} |
| self.mr = mr_remote_function[self.remote_operation](self.msg_size + GRH_SIZE) |
| |
| def create_event_channel(self): |
| self.channel = CMEventChannel() |
| |
| def create_qp_init_attr(self, rcq=None, scq=None): |
| return QPInitAttr(qp_type=self.qp_type, rcq=rcq, scq=scq, |
| cap=QPCap(max_recv_wr=1)) |
| |
| def create_conn_param(self, qp_num=0, conn_idx=0): |
| if self.with_ext_qp: |
| qp_num = self.qp.qp_num |
| return ConnParam(qp_num=qp_num) |
| |
| def set_ud_params(self, cm_event): |
| if self.port_space in [ce.RDMA_PS_UDP, ce.RDMA_PS_IPOIB]: |
| self.ud_params = UDParam(cm_event) |
| |
| def my_qp_number(self): |
| if self.with_ext_qp: |
| return self.qp.qp_num |
| else: |
| cm = self.child_id if self.passive else self.cmid |
| return cm.qpn |
| |
| def create_qp(self, conn_idx=0): |
| """ |
| Create an rdmacm QP. If self.with_ext_qp is set, then an external CQ and |
| QP will be created. In case that CQ is already created, it is used |
| for the newly created QP. |
| :param conn_idx: The connection index. |
| """ |
| cmid = self.child_id if self.passive else self.cmid |
| if not self.with_ext_qp: |
| cmid.create_qp(self.create_qp_init_attr()) |
| else: |
| self.create_cq(cmid) |
| init_attr = self.create_qp_init_attr(rcq=self.cq, scq=self.cq) |
| self.qps[conn_idx] = QP(cmid.pd, init_attr, QPAttr()) |
| |
| def create_cq(self, cmid): |
| if not self.cq: |
| self.cq = CQ(cmid.context, self.num_msgs, None, None, 0) |
| |
| def modify_ext_qp_to_rts(self, conn_idx=0): |
| cmid = self.child_id if self.passive else self.cmid |
| attr, mask = cmid.init_qp_attr(e.IBV_QPS_INIT) |
| self.qps[conn_idx].modify(attr, mask) |
| attr, mask = cmid.init_qp_attr(e.IBV_QPS_RTR) |
| self.qps[conn_idx].modify(attr, mask) |
| attr, mask = cmid.init_qp_attr(e.IBV_QPS_RTS) |
| self.qps[conn_idx].modify(attr, mask) |
| |
| def mem_write(self, data, size, offset=0): |
| self.mr.write(data, size, offset) |
| |
| def mem_read(self, size=None, offset=0): |
| size_ = self.msg_size if size is None else size |
| return self.mr.read(size_, offset) |
| |
| @abc.abstractmethod |
| def create_child_id(self, cm_event=None): |
| pass |
| |
| @property |
| def mr_lkey(self): |
| return self.mr.lkey |
| |
| |
| class AsyncCMResources(CMResources): |
| """ |
| AsyncCMResources class contains resources for RDMA CM asynchronous |
| communication. |
| :param addr: Local address to bind to. |
| :param passive: Indicate if this CM is the passive CM. |
| """ |
| def __init__(self, addr=None, passive=None, **kwargs): |
| super(AsyncCMResources, self).__init__(addr=addr, passive=passive, |
| **kwargs) |
| self.create_event_channel() |
| |
| def create_cmid(self, idx=0): |
| self.cmids[idx] = CMID(creator=self.channel, port_space=self.port_space) |
| |
| def create_child_id(self, cm_event=None): |
| if not self.passive: |
| raise PyverbsUserError('create_child_id can be used only in passive side') |
| new_child_idx = len(self.child_ids) |
| self.child_ids[new_child_idx] = CMID(creator=cm_event, listen_id=self.cmid) |
| |
| |
| class SyncCMResources(CMResources): |
| """ |
| SyncCMResources class contains resources for RDMA CM synchronous |
| communication. |
| :param addr: Local address to bind to. |
| :param passive: Indicate if this CM is the passive CM. |
| """ |
| def __init__(self, addr=None, passive=None, **kwargs): |
| super(SyncCMResources, self).__init__(addr=addr, passive=passive, |
| **kwargs) |
| |
| def create_cmid(self, idx=0): |
| self.cmids[idx] = CMID(creator=self.ai, qp_init_attr=self.qp_init_attr) |
| |
| def create_child_id(self, cm_event=None): |
| if not self.passive: |
| raise PyverbsUserError('create_child_id can be used only in passive side') |
| new_child_idx = len(self.child_ids) |
| self.child_ids[new_child_idx] = self.cmid.get_request() |
