| # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) |
| # Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. |
| |
| from libc.stdlib cimport malloc, free |
| from libc.string cimport memcpy |
| import weakref |
| |
| from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError, PyverbsRDMAError |
| from pyverbs.utils import gid_str, qp_type_to_str, qp_state_to_str, mtu_to_str |
| from pyverbs.utils import access_flags_to_str, mig_state_to_str |
| from pyverbs.wq cimport RwqIndTable, RxHashConf |
| from pyverbs.mr cimport MW, MWBindInfo, MWBind |
| from pyverbs.wr cimport RecvWR, SendWR, SGE |
| from pyverbs.base import PyverbsRDMAErrno |
| from pyverbs.addr cimport AHAttr, GID, AH |
| from pyverbs.flow cimport FlowAttr, Flow |
| from pyverbs.base cimport close_weakrefs |
| cimport pyverbs.libibverbs_enums as e |
| from pyverbs.addr cimport GlobalRoute |
| from pyverbs.device cimport Context |
| from cpython.ref cimport PyObject |
| from pyverbs.cq cimport CQ, CQEX |
| cimport pyverbs.libibverbs as v |
| from pyverbs.xrcd cimport XRCD |
| from pyverbs.srq cimport SRQ |
| from pyverbs.pd cimport PD |
| |
| cdef extern from 'Python.h': |
| void* PyLong_AsVoidPtr(object) |
| cdef extern from 'endian.h': |
| unsigned long htobe32(unsigned long host_32bits) |
| |
| |
| cdef class QPCap(PyverbsObject): |
| def __init__(self, max_send_wr=1, max_recv_wr=10, max_send_sge=1, |
| max_recv_sge=1, max_inline_data=0): |
| """ |
| Initializes a QPCap object with user-provided or default values. |
| :param max_send_wr: max number of outstanding WRs in the SQ |
| :param max_recv_wr: max number of outstanding WRs in the RQ |
| :param max_send_sge: Requested max number of scatter-gather elements in |
| a WR in the SQ |
| :param max_recv_sge: Requested max number of scatter-gather elements in |
| a WR in the RQ |
| :param max_inline_data: max number of data (bytes) that can be posted |
| inline to the SQ, otherwise 0 |
| :return: |
| """ |
| super().__init__() |
| self.cap.max_send_wr = max_send_wr |
| self.cap.max_recv_wr = max_recv_wr |
| self.cap.max_send_sge = max_send_sge |
| self.cap.max_recv_sge = max_recv_sge |
| self.cap.max_inline_data = max_inline_data |
| |
| @property |
| def max_send_wr(self): |
| return self.cap.max_send_wr |
| @max_send_wr.setter |
| def max_send_wr(self, val): |
| self.cap.max_send_wr = val |
| |
| @property |
| def max_recv_wr(self): |
| return self.cap.max_recv_wr |
| @max_recv_wr.setter |
| def max_recv_wr(self, val): |
| self.cap.max_recv_wr = val |
| |
| @property |
| def max_send_sge(self): |
| return self.cap.max_send_sge |
| @max_send_sge.setter |
| def max_send_sge(self, val): |
| self.cap.max_send_sge = val |
| |
| @property |
| def max_recv_sge(self): |
| return self.cap.max_recv_sge |
| @max_recv_sge.setter |
| def max_recv_sge(self, val): |
| self.cap.max_recv_sge = val |
| |
| @property |
| def max_inline_data(self): |
| return self.cap.max_inline_data |
| @max_inline_data.setter |
| def max_inline_data(self, val): |
| self.cap.max_inline_data = val |
| |
| def __str__(self): |
| print_format = '{:20}: {:<20}\n' |
| return print_format.format('max send wrs', self.cap.max_send_wr) +\ |
| print_format.format('max recv wrs', self.cap.max_recv_wr) +\ |
| print_format.format('max send sges', self.cap.max_send_sge) +\ |
| print_format.format('max recv sges', self.cap.max_recv_sge) +\ |
| print_format.format('max inline data', self.cap.max_inline_data) |
| |
| |
| cdef class QPInitAttr(PyverbsObject): |
| def __init__(self, qp_type=e.IBV_QPT_UD, qp_context=None, |
| PyverbsObject scq=None, PyverbsObject rcq=None, |
| SRQ srq=None, QPCap cap=None, sq_sig_all=1): |
| """ |
| Initializes a QpInitAttr object representing ibv_qp_init_attr struct. |
| Note that SRQ object is not yet supported in pyverbs so can't be passed |
| as a parameter. None should be used until such support is added. |
| :param qp_type: The desired QP type (see enum ibv_qp_type) |
| :param qp_context: Associated QP context |
| :param scq: Send CQ to be used for this QP |
| :param rcq: Receive CQ to be used for this QP |
| :param srq: Shared receive queue to be used as RQ in QP |
| :param cap: A QPCap object |
| :param sq_sig_all: If set, each send WR will generate a completion |
| entry |
| :return: A QpInitAttr object |
| """ |
| super().__init__() |
| _copy_caps(cap, self) |
| self.attr.qp_context = <void*>qp_context |
| if scq is not None: |
| if type(scq) is CQ: |
| self.attr.send_cq = (<CQ>scq).cq |
| elif isinstance(scq, CQEX): |
| self.attr.send_cq = (<CQEX>scq).ibv_cq |
| else: |
| raise PyverbsUserError('Expected CQ/CQEX, got {t}'.\ |
| format(t=type(scq))) |
| self.scq = scq |
| |
| if rcq is not None: |
| if type(rcq) is CQ: |
| self.attr.recv_cq = (<CQ>rcq).cq |
| elif isinstance(rcq, CQEX): |
| self.attr.recv_cq = (<CQEX>rcq).ibv_cq |
| else: |
| raise PyverbsUserError('Expected CQ/CQEX, got {t}'.\ |
| format(t=type(rcq))) |
| self.rcq = rcq |
| self.attr.qp_type = qp_type |
| self.attr.sq_sig_all = sq_sig_all |
| self.srq = srq |
| self.attr.srq = srq.srq if srq else NULL |
| |
| @property |
| def send_cq(self): |
| return self.scq |
| @send_cq.setter |
| def send_cq(self, val): |
| if type(val) is CQ: |
| self.attr.send_cq = (<CQ>val).cq |
| elif type(val) is CQEX: |
| self.attr.send_cq = (<CQEX>val).ibv_cq |
| self.scq = val |
| |
| @property |
| def srq(self): |
| return self.srq |
| @srq.setter |
| def srq(self, SRQ val): |
| self.attr.srq = <v.ibv_srq*>val.srq |
| self.srq = val |
| |
| @property |
| def recv_cq(self): |
| return self.rcq |
| @recv_cq.setter |
| def recv_cq(self, val): |
| if type(val) is CQ: |
| self.attr.recv_cq = (<CQ>val).cq |
| elif type(val) is CQEX: |
| self.attr.recv_cq = (<CQEX>val).ibv_cq |
| self.rcq = val |
| |
| @property |
| def cap(self): |
| return QPCap(max_send_wr=self.attr.cap.max_send_wr, |
| max_recv_wr=self.attr.cap.max_recv_wr, |
| max_send_sge=self.attr.cap.max_send_sge, |
| max_recv_sge=self.attr.cap.max_recv_sge, |
| max_inline_data=self.attr.cap.max_inline_data) |
| @cap.setter |
| def cap(self, val): |
| _copy_caps(val, self) |
| |
| @property |
| def qp_type(self): |
| return self.attr.qp_type |
| @qp_type.setter |
| def qp_type(self, val): |
| self.attr.qp_type = val |
| |
| @property |
| def sq_sig_all(self): |
| return self.attr.sq_sig_all |
| @sq_sig_all.setter |
| def sq_sig_all(self, val): |
| self.attr.sq_sig_all = val |
| |
| @property |
| def max_send_wr(self): |
| return self.attr.cap.max_send_wr |
| @max_send_wr.setter |
| def max_send_wr(self, val): |
| self.attr.cap.max_send_wr = val |
| |
| @property |
| def max_recv_wr(self): |
| return self.attr.cap.max_recv_wr |
| @max_recv_wr.setter |
| def max_recv_wr(self, val): |
| self.attr.cap.max_recv_wr = val |
| |
| @property |
| def max_send_sge(self): |
| return self.attr.cap.max_send_sge |
| @max_send_sge.setter |
| def max_send_sge(self, val): |
| self.attr.cap.max_send_sge = val |
| |
| @property |
| def max_recv_sge(self): |
| return self.attr.cap.max_recv_sge |
| @max_recv_sge.setter |
| def max_recv_sge(self, val): |
| self.attr.cap.max_recv_sge = val |
| |
| @property |
| def max_inline_data(self): |
| return self.attr.cap.max_inline_data |
| @max_inline_data.setter |
| def max_inline_data(self, val): |
| self.attr.cap.max_inline_data = val |
| |
| def __str__(self): |
| print_format = '{:20}: {:<20}\n' |
| ident_format = ' {:20}: {:<20}\n' |
| return print_format.format('QP type', qp_type_to_str(self.qp_type)) +\ |
| print_format.format('SQ sig. all', self.sq_sig_all) +\ |
| 'QP caps:\n' +\ |
| ident_format.format('max send WR', self.attr.cap.max_send_wr) +\ |
| ident_format.format('max recv WR', self.attr.cap.max_recv_wr) +\ |
| ident_format.format('max send SGE', |
| self.attr.cap.max_send_sge) +\ |
| ident_format.format('max recv SGE', |
| self.attr.cap.max_recv_sge) +\ |
| ident_format.format('max inline data', |
| self.attr.cap.max_inline_data) |
| |
| |
| cdef class QPInitAttrEx(PyverbsObject): |
| def __init__(self, qp_type=e.IBV_QPT_UD, qp_context=None, |
| PyverbsObject scq=None, PyverbsObject rcq=None, |
| SRQ srq=None, QPCap cap=None, sq_sig_all=0, comp_mask=0, |
| PD pd=None, XRCD xrcd=None, create_flags=0, |
| max_tso_header=0, source_qpn=0, RxHashConf hash_conf=None, |
| RwqIndTable ind_table=None, send_ops_flags=0): |
| """ |
| Initialize a QPInitAttrEx object with user-defined or default values. |
| :param qp_type: QP type to be created |
| :param qp_context: Associated user context |
| :param scq: Send CQ to be used for this QP |
| :param rcq: Recv CQ to be used for this QP |
| :param srq: Shared receive queue to be used as RQ in QP |
| :param cap: A QPCap object |
| :param sq_sig_all: If set, each send WR will generate a completion |
| entry |
| :param comp_mask: bit mask to determine which of the following fields |
| are valid |
| :param pd: A PD object to be associated with this QP |
| :param xrcd: XRC domain to be used for XRC QPs |
| :param create_flags: Creation flags for this QP |
| :param max_tso_header: Maximum TSO header size |
| :param source_qpn: Source QP number (requires IBV_QP_CREATE_SOURCE_QPN |
| set in create_flags) |
| :param hash_conf: A RxHashConf object, config of RX hash key. |
| :param ind_table: A RwqIndTable object, indirection table of RWQs. |
| :param send_ops_flags: Send opcodes to be supported by the extended QP. |
| Use ibv_qp_create_send_ops_flags enum |
| :return: An initialized QPInitAttrEx object |
| """ |
| super().__init__() |
| _copy_caps(cap, self) |
| if scq is not None: |
| if type(scq) is CQ: |
| self.attr.send_cq = (<CQ>scq).cq |
| elif isinstance(scq, CQEX): |
| self.attr.send_cq = (<CQEX>scq).ibv_cq |
| else: |
| raise PyverbsUserError('Expected CQ/CQEX, got {t}'.\ |
| format(t=type(scq))) |
| self.scq = scq |
| |
| if rcq is not None: |
| if type(rcq) is CQ: |
| self.attr.recv_cq = (<CQ>rcq).cq |
| elif isinstance(rcq, CQEX): |
| self.attr.recv_cq = (<CQEX>rcq).ibv_cq |
| else: |
| raise PyverbsUserError('Expected CQ/CQEX, got {t}'.\ |
| format(t=type(rcq))) |
| self.rcq = rcq |
| |
| self.srq = srq |
| self.attr.srq = srq.srq if srq else NULL |
| self.xrcd = xrcd |
| self.attr.xrcd = xrcd.xrcd if xrcd else NULL |
| if hash_conf: |
| self.attr.rx_hash_conf = hash_conf.rx_hash_conf |
| self.ind_table = ind_table |
| self.attr.rwq_ind_tbl = ind_table.rwq_ind_table if ind_table else NULL |
| self.attr.qp_type = qp_type |
| self.attr.sq_sig_all = sq_sig_all |
| self.attr.comp_mask = comp_mask |
| if pd is not None: |
| self._pd = pd |
| self.attr.pd = pd.pd |
| self.attr.create_flags = create_flags |
| self.attr.max_tso_header = max_tso_header |
| self.attr.source_qpn = source_qpn |
| self.attr.send_ops_flags = send_ops_flags |
| |
| @property |
| def send_cq(self): |
| return self.scq |
| @send_cq.setter |
| def send_cq(self, val): |
| if type(val) is CQ: |
| self.attr.send_cq = (<CQ>val).cq |
| elif type(val) is CQEX: |
| self.attr.send_cq = (<CQEX>val).ibv_cq |
| self.scq = val |
| |
| @property |
| def recv_cq(self): |
| return self.rcq |
| @recv_cq.setter |
| def recv_cq(self, val): |
| if type(val) is CQ: |
| self.attr.recv_cq = (<CQ>val).cq |
| elif type(val) is CQEX: |
| self.attr.recv_cq = (<CQEX>val).ibv_cq |
| self.rcq = val |
| |
| @property |
| def cap(self): |
| return QPCap(max_send_wr=self.attr.cap.max_send_wr, |
| max_recv_wr=self.attr.cap.max_recv_wr, |
| max_send_sge=self.attr.cap.max_send_sge, |
| max_recv_sge=self.attr.cap.max_recv_sge, |
| max_inline_data=self.attr.cap.max_inline_data) |
| @cap.setter |
| def cap(self, val): |
| _copy_caps(val, self) |
| |
| @property |
| def qp_type(self): |
| return self.attr.qp_type |
| @qp_type.setter |
| def qp_type(self, val): |
| self.attr.qp_type = val |
| |
| @property |
| def sq_sig_all(self): |
| return self.attr.sq_sig_all |
| @sq_sig_all.setter |
| def sq_sig_all(self, val): |
| self.attr.sq_sig_all = val |
| |
| @property |
| def comp_mask(self): |
| return self.attr.comp_mask |
| @comp_mask.setter |
| def comp_mask(self, val): |
| self.attr.comp_mask = val |
| |
| @property |
| def pd(self): |
| return self._pd |
| @pd.setter |
| def pd(self, PD val): |
| self.attr.pd = <v.ibv_pd*>val.pd |
| self._pd = val |
| |
| @property |
| def xrcd(self): |
| return self.xrcd |
| @xrcd.setter |
| def xrcd(self, XRCD val): |
| self.attr.xrcd = <v.ibv_xrcd*>val.xrcd |
| self.xrcd = val |
| |
| @property |
| def srq(self): |
| return self.srq |
| @srq.setter |
| def srq(self, SRQ val): |
| self.attr.srq = <v.ibv_srq*>val.srq |
| self.srq = val |
| |
| @property |
| def create_flags(self): |
| return self.attr.create_flags |
| @create_flags.setter |
| def create_flags(self, val): |
| self.attr.create_flags = val |
| |
| @property |
| def max_tso_header(self): |
| return self.attr.max_tso_header |
| @max_tso_header.setter |
| def max_tso_header(self, val): |
| self.attr.max_tso_header = val |
| |
| @property |
| def source_qpn(self): |
| return self.attr.source_qpn |
| @source_qpn.setter |
| def source_qpn(self, val): |
| self.attr.source_qpn = val |
| |
| @property |
| def max_send_wr(self): |
| return self.attr.cap.max_send_wr |
| @max_send_wr.setter |
| def max_send_wr(self, val): |
| self.attr.cap.max_send_wr = val |
| |
| @property |
| def max_recv_wr(self): |
| return self.attr.cap.max_recv_wr |
| @max_recv_wr.setter |
| def max_recv_wr(self, val): |
| self.attr.cap.max_recv_wr = val |
| |
| @property |
| def max_send_sge(self): |
| return self.attr.cap.max_send_sge |
| @max_send_sge.setter |
| def max_send_sge(self, val): |
| self.attr.cap.max_send_sge = val |
| |
| @property |
| def max_recv_sge(self): |
| return self.attr.cap.max_recv_sge |
| @max_recv_sge.setter |
| def max_recv_sge(self, val): |
| self.attr.cap.max_recv_sge = val |
| |
| @property |
| def max_inline_data(self): |
| return self.attr.cap.max_inline_data |
| @max_inline_data.setter |
| def max_inline_data(self, val): |
| self.attr.cap.max_inline_data = val |
| |
| @property |
| def ind_table(self): |
| return self.ind_table |
| @ind_table.setter |
| def ind_table(self, RwqIndTable val): |
| self.attr.rwq_ind_tbl = <v.ibv_rwq_ind_table*>val.rwq_ind_table |
| self.ind_table = val |
| |
| def mask_to_str(self, mask): |
| comp_masks = {1: 'PD', 2: 'XRCD', 4: 'Create Flags', |
| 8: 'Max TSO header', 16: 'Indirection Table', |
| 32: 'RX hash'} |
| mask_str = '' |
| for f in comp_masks: |
| if mask & f: |
| mask_str += comp_masks[f] |
| mask_str += ' ' |
| return mask_str |
| |
| def flags_to_str(self, flags): |
| create_flags = {1: 'Block self mcast loopback', 2: 'Scatter FCS', |
| 4: 'CVLAN stripping', 8: 'Source QPN', |
| 16: 'PCI write end padding'} |
| create_str = '' |
| for f in create_flags: |
| if flags & f: |
| create_str += create_flags[f] |
| create_str += ' ' |
| return create_str |
| |
| def __str__(self): |
| print_format = '{:20}: {:<20}\n' |
| return print_format.format('QP type', qp_type_to_str(self.qp_type)) +\ |
| print_format.format('SQ sig. all', self.sq_sig_all) +\ |
| 'QP caps:\n' +\ |
| print_format.format(' max send WR', |
| self.attr.cap.max_send_wr) +\ |
| print_format.format(' max recv WR', |
| self.attr.cap.max_recv_wr) +\ |
| print_format.format(' max send SGE', |
| self.attr.cap.max_send_sge) +\ |
| print_format.format(' max recv SGE', |
| self.attr.cap.max_recv_sge) +\ |
| print_format.format(' max inline data', |
| self.attr.cap.max_inline_data) +\ |
| print_format.format('comp mask', |
| self.mask_to_str(self.attr.comp_mask)) +\ |
| print_format.format('create flags', |
| self.flags_to_str(self.attr.create_flags)) +\ |
| print_format.format('max TSO header', |
| self.attr.max_tso_header) +\ |
| print_format.format('Source QPN', self.attr.source_qpn) |
| |
| |
| cdef class QPAttr(PyverbsObject): |
| def __init__(self, qp_state=e.IBV_QPS_INIT, cur_qp_state=e.IBV_QPS_RESET, |
| port_num=1, path_mtu=e.IBV_MTU_1024): |
| """ |
| Initializes a QPQttr object which represents ibv_qp_attr structs. It |
| can be used to modify a QP. |
| This function initializes default values for reset-to-init transition. |
| :param qp_state: Desired QP state |
| :param cur_qp_state: Current QP state |
| :return: An initialized QpAttr object |
| """ |
| super().__init__() |
| self.attr.qp_state = qp_state |
| self.attr.cur_qp_state = cur_qp_state |
| self.attr.port_num = port_num |
| self.attr.path_mtu = path_mtu |
| |
| @property |
| def qp_state(self): |
| return self.attr.qp_state |
| @qp_state.setter |
| def qp_state(self, val): |
| self.attr.qp_state = val |
| |
| @property |
| def cur_qp_state(self): |
| return self.attr.cur_qp_state |
| @cur_qp_state.setter |
| def cur_qp_state(self, val): |
| self.attr.cur_qp_state = val |
| |
| @property |
| def path_mtu(self): |
| return self.attr.path_mtu |
| @path_mtu.setter |
| def path_mtu(self, val): |
| self.attr.path_mtu = val |
| |
| @property |
| def path_mig_state(self): |
| return self.attr.path_mig_state |
| @path_mig_state.setter |
| def path_mig_state(self, val): |
| self.attr.path_mig_state = val |
| |
| @property |
| def qkey(self): |
| return self.attr.qkey |
| @qkey.setter |
| def qkey(self, val): |
| self.attr.qkey = val |
| |
| @property |
| def rq_psn(self): |
| return self.attr.rq_psn |
| @rq_psn.setter |
| def rq_psn(self, val): |
| self.attr.rq_psn = val |
| |
| @property |
| def sq_psn(self): |
| return self.attr.sq_psn |
| @sq_psn.setter |
| def sq_psn(self, val): |
| self.attr.sq_psn = val |
| |
| @property |
| def dest_qp_num(self): |
| return self.attr.dest_qp_num |
| @dest_qp_num.setter |
| def dest_qp_num(self, val): |
| self.attr.dest_qp_num = val |
| |
| @property |
| def qp_access_flags(self): |
| return self.attr.qp_access_flags |
| @qp_access_flags.setter |
| def qp_access_flags(self, val): |
| self.attr.qp_access_flags = val |
| |
| @property |
| def cap(self): |
| return QPCap(max_send_wr=self.attr.cap.max_send_wr, |
| max_recv_wr=self.attr.cap.max_recv_wr, |
| max_send_sge=self.attr.cap.max_send_sge, |
| max_recv_sge=self.attr.cap.max_recv_sge, |
| max_inline_data=self.attr.cap.max_inline_data) |
| @cap.setter |
| def cap(self, val): |
| _copy_caps(val, self) |
| |
| @property |
| def ah_attr(self): |
| if self.attr.ah_attr.is_global: |
| gid = gid_str(self.attr.ah_attr.grh.dgid._global.subnet_prefix, |
| self.attr.ah_attr.grh.dgid._global.interface_id) |
| g = GID(gid) |
| gr = GlobalRoute(flow_label=self.attr.ah_attr.grh.flow_label, |
| sgid_index=self.attr.ah_attr.grh.sgid_index, |
| hop_limit=self.attr.ah_attr.grh.hop_limit, dgid=g, |
| traffic_class=self.attr.ah_attr.grh.traffic_class) |
| else: |
| gr = None |
| ah = AHAttr(dlid=self.attr.ah_attr.dlid, sl=self.attr.ah_attr.sl, |
| port_num=self.attr.ah_attr.port_num, |
| src_path_bits=self.attr.ah_attr.src_path_bits, |
| static_rate=self.attr.ah_attr.static_rate, |
| is_global=self.attr.ah_attr.is_global, gr=gr) |
| return ah |
| |
| @ah_attr.setter |
| def ah_attr(self, val): |
| self._copy_ah(val) |
| |
| @property |
| def alt_ah_attr(self): |
| if self.attr.alt_ah_attr.is_global: |
| gid = gid_str(self.attr.alt_ah_attr.grh.dgid._global.subnet_prefix, |
| self.attr.alt_ah_attr.grh.dgid._global.interface_id) |
| g = GID(gid) |
| gr = GlobalRoute(flow_label=self.attr.alt_ah_attr.grh.flow_label, |
| sgid_index=self.attr.alt_ah_attr.grh.sgid_index, |
| hop_limit=self.attr.alt_ah_attr.grh.hop_limit, |
| dgid=g, |
| traffic_class=self.attr.alt_ah_attr.grh.traffic_class) |
| else: |
| gr = None |
| ah = AHAttr(dlid=self.attr.alt_ah_attr.dlid, |
| port_num=self.attr.ah_attr.port_num, |
| sl=self.attr.alt_ah_attr.sl, |
| src_path_bits=self.attr.alt_ah_attr.src_path_bits, |
| static_rate=self.attr.alt_ah_attr.static_rate, |
| is_global=self.attr.alt_ah_attr.is_global, gr=gr) |
| return ah |
| |
| @alt_ah_attr.setter |
| def alt_ah_attr(self, val): |
| self._copy_ah(val, True) |
| |
| def _copy_ah(self, AHAttr ah_attr, is_alt=False): |
| if ah_attr is None: |
| return |
| if not is_alt: |
| for i in range(16): |
| self.attr.ah_attr.grh.dgid.raw[i] = \ |
| ah_attr.ah_attr.grh.dgid.raw[i] |
| self.attr.ah_attr.grh.flow_label = ah_attr.ah_attr.grh.flow_label |
| self.attr.ah_attr.grh.sgid_index = ah_attr.ah_attr.grh.sgid_index |
| self.attr.ah_attr.grh.hop_limit = ah_attr.ah_attr.grh.hop_limit |
| self.attr.ah_attr.grh.traffic_class = \ |
| ah_attr.ah_attr.grh.traffic_class |
| self.attr.ah_attr.dlid = ah_attr.ah_attr.dlid |
| self.attr.ah_attr.sl = ah_attr.ah_attr.sl |
| self.attr.ah_attr.src_path_bits = ah_attr.ah_attr.src_path_bits |
| self.attr.ah_attr.static_rate = ah_attr.ah_attr.static_rate |
| self.attr.ah_attr.is_global = ah_attr.ah_attr.is_global |
| self.attr.ah_attr.port_num = ah_attr.ah_attr.port_num |
| else: |
| for i in range(16): |
| self.attr.alt_ah_attr.grh.dgid.raw[i] = \ |
| ah_attr.ah_attr.grh.dgid.raw[i] |
| self.attr.alt_ah_attr.grh.flow_label = \ |
| ah_attr.ah_attr.grh.flow_label |
| self.attr.alt_ah_attr.grh.sgid_index = \ |
| ah_attr.ah_attr.grh.sgid_index |
| self.attr.alt_ah_attr.grh.hop_limit = ah_attr.ah_attr.grh.hop_limit |
| self.attr.alt_ah_attr.grh.traffic_class = \ |
| ah_attr.ah_attr.grh.traffic_class |
| self.attr.alt_ah_attr.dlid = ah_attr.ah_attr.dlid |
| self.attr.alt_ah_attr.sl = ah_attr.ah_attr.sl |
| self.attr.alt_ah_attr.src_path_bits = ah_attr.ah_attr.src_path_bits |
| self.attr.alt_ah_attr.static_rate = ah_attr.ah_attr.static_rate |
| self.attr.alt_ah_attr.is_global = ah_attr.ah_attr.is_global |
| self.attr.alt_ah_attr.port_num = ah_attr.ah_attr.port_num |
| |
| @property |
| def pkey_index(self): |
| return self.attr.pkey_index |
| @pkey_index.setter |
| def pkey_index(self, val): |
| self.attr.pkey_index = val |
| |
| @property |
| def alt_pkey_index(self): |
| return self.attr.alt_pkey_index |
| @alt_pkey_index.setter |
| def alt_pkey_index(self, val): |
| self.attr.alt_pkey_index = val |
| |
| @property |
| def en_sqd_async_notify(self): |
| return self.attr.en_sqd_async_notify |
| @en_sqd_async_notify.setter |
| def en_sqd_async_notify(self, val): |
| self.attr.en_sqd_async_notify = val |
| |
| @property |
| def sq_draining(self): |
| return self.attr.sq_draining |
| @sq_draining.setter |
| def sq_draining(self, val): |
| self.attr.sq_draining = val |
| |
| @property |
| def max_rd_atomic(self): |
| return self.attr.max_rd_atomic |
| @max_rd_atomic.setter |
| def max_rd_atomic(self, val): |
| self.attr.max_rd_atomic = val |
| |
| @property |
| def max_dest_rd_atomic(self): |
| return self.attr.max_dest_rd_atomic |
| @max_dest_rd_atomic.setter |
| def max_dest_rd_atomic(self, val): |
| self.attr.max_dest_rd_atomic = val |
| |
| @property |
| def min_rnr_timer(self): |
| return self.attr.min_rnr_timer |
| @min_rnr_timer.setter |
| def min_rnr_timer(self, val): |
| self.attr.min_rnr_timer = val |
| |
| @property |
| def port_num(self): |
| return self.attr.port_num |
| @port_num.setter |
| def port_num(self, val): |
| self.attr.port_num = val |
| |
| @property |
| def timeout(self): |
| return self.attr.timeout |
| @timeout.setter |
| def timeout(self, val): |
| self.attr.timeout = val |
| |
| @property |
| def retry_cnt(self): |
| return self.attr.retry_cnt |
| @retry_cnt.setter |
| def retry_cnt(self, val): |
| self.attr.retry_cnt = val |
| |
| @property |
| def rnr_retry(self): |
| return self.attr.rnr_retry |
| @rnr_retry.setter |
| def rnr_retry(self, val): |
| self.attr.rnr_retry = val |
| |
| @property |
| def alt_port_num(self): |
| return self.attr.alt_port_num |
| @alt_port_num.setter |
| def alt_port_num(self, val): |
| self.attr.alt_port_num = val |
| |
| @property |
| def alt_timeout(self): |
| return self.attr.alt_timeout |
| @alt_timeout.setter |
| def alt_timeout(self, val): |
| self.attr.alt_timeout = val |
| |
| @property |
| def rate_limit(self): |
| return self.attr.rate_limit |
| @rate_limit.setter |
| def rate_limit(self, val): |
| self.attr.rate_limit = val |
| |
| def __str__(self): |
| print_format = '{:22}: {:<20}\n' |
| ah_format = ' {:22}: {:<20}\n' |
| ident_format = ' {:22}: {:<20}\n' |
| if self.attr.ah_attr.is_global: |
| global_ah = ah_format.format('dgid', |
| gid_str(self.attr.ah_attr.grh.dgid._global.subnet_prefix, |
| self.attr.ah_attr.grh.dgid._global.interface_id)) +\ |
| ah_format.format('flow label', |
| self.attr.ah_attr.grh.flow_label) +\ |
| ah_format.format('sgid index', |
| self.attr.ah_attr.grh.sgid_index) +\ |
| ah_format.format('hop limit', |
| self.attr.ah_attr.grh.hop_limit) +\ |
| ah_format.format('traffic_class', |
| self.attr.ah_attr.grh.traffic_class) |
| else: |
| global_ah = '' |
| if self.attr.alt_ah_attr.is_global: |
| alt_global_ah = ah_format.format('dgid', |
| gid_str(self.attr.alt_ah_attr.grh.dgid._global.subnet_prefix, |
| self.attr.alt_ah_attr.grh.dgid._global.interface_id)) +\ |
| ah_format.format('flow label', |
| self.attr.alt_ah_attr.grh.flow_label) +\ |
| ah_format.format('sgid index', |
| self.attr.alt_ah_attr.grh.sgid_index) +\ |
| ah_format.format('hop limit', |
| self.attr.alt_ah_attr.grh.hop_limit) +\ |
| ah_format.format('traffic_class', |
| self.attr.alt_ah_attr.grh.traffic_class) |
| else: |
| alt_global_ah = '' |
| return print_format.format('QP state', |
| qp_state_to_str(self.attr.qp_state)) +\ |
| print_format.format('QP current state', |
| qp_state_to_str(self.attr.cur_qp_state)) +\ |
| print_format.format('Path MTU', |
| mtu_to_str(self.attr.path_mtu)) +\ |
| print_format.format('Path mig. state', |
| mig_state_to_str(self.attr.path_mig_state)) +\ |
| print_format.format('QKey', self.attr.qkey) +\ |
| print_format.format('RQ PSN', self.attr.rq_psn) +\ |
| print_format.format('SQ PSN', self.attr.sq_psn) +\ |
| print_format.format('Dest QP number', self.attr.dest_qp_num) +\ |
| print_format.format('QP access flags', |
| access_flags_to_str(self.attr.qp_access_flags)) +\ |
| 'QP caps:\n' +\ |
| ident_format.format('max send WR', |
| self.attr.cap.max_send_wr) +\ |
| ident_format.format('max recv WR', |
| self.attr.cap.max_recv_wr) +\ |
| ident_format.format('max send SGE', |
| self.attr.cap.max_send_sge) +\ |
| ident_format.format('max recv SGE', |
| self.attr.cap.max_recv_sge) +\ |
| ident_format.format('max inline data', |
| self.attr.cap.max_inline_data) +\ |
| 'AH Attr:\n' +\ |
| ident_format.format('port num', self.attr.ah_attr.port_num) +\ |
| ident_format.format('sl', self.attr.ah_attr.sl) +\ |
| ident_format.format('source path bits', |
| self.attr.ah_attr.src_path_bits) +\ |
| ident_format.format('dlid', self.attr.ah_attr.dlid) +\ |
| ident_format.format('port num', self.attr.ah_attr.port_num) +\ |
| ident_format.format('static rate', |
| self.attr.ah_attr.static_rate) +\ |
| ident_format.format('is global', |
| self.attr.ah_attr.is_global) +\ |
| global_ah +\ |
| 'Alt. AH Attr:\n' +\ |
| ident_format.format('port num', self.attr.alt_ah_attr.port_num) +\ |
| ident_format.format('sl', self.attr.alt_ah_attr.sl) +\ |
| ident_format.format('source path bits', |
| self.attr.alt_ah_attr.src_path_bits) +\ |
| ident_format.format('dlid', self.attr.alt_ah_attr.dlid) +\ |
| ident_format.format('port num', self.attr.alt_ah_attr.port_num) +\ |
| ident_format.format('static rate', |
| self.attr.alt_ah_attr.static_rate) +\ |
| ident_format.format('is global', |
| self.attr.alt_ah_attr.is_global) +\ |
| alt_global_ah +\ |
| print_format.format('PKey index', self.attr.pkey_index) +\ |
| print_format.format('Alt. PKey index', |
| self.attr.alt_pkey_index) +\ |
| print_format.format('En. SQD async notify', |
| self.attr.en_sqd_async_notify) +\ |
| print_format.format('SQ draining', self.attr.sq_draining) +\ |
| print_format.format('Max RD atomic', self.attr.max_rd_atomic) +\ |
| print_format.format('Max dest. RD atomic', |
| self.attr.max_dest_rd_atomic) +\ |
| print_format.format('Min RNR timer', self.attr.min_rnr_timer) +\ |
| print_format.format('Port number', self.attr.port_num) +\ |
| print_format.format('Timeout', self.attr.timeout) +\ |
| print_format.format('Retry counter', self.attr.retry_cnt) +\ |
| print_format.format('RNR retry', self.attr.rnr_retry) +\ |
| print_format.format('Alt. port number', |
| self.attr.alt_port_num) +\ |
| print_format.format('Alt. timeout', self.attr.alt_timeout) +\ |
| print_format.format('Rate limit', self.attr.rate_limit) |
| |
| |
| cdef class ECE(PyverbsCM): |
| def __init__(self, vendor_id=0, options=0, comp_mask=0): |
| """ |
| :param vendor_id: Unique identifier of the provider vendor. |
| :param options: Provider specific attributes which are supported or |
| needed to be enabled by ECE users. |
| :param comp_mask: A bitmask specifying which ECE options should be |
| valid. |
| """ |
| super().__init__() |
| self.ece.vendor_id = vendor_id |
| self.ece.options = options |
| self.ece.comp_mask = comp_mask |
| |
| @property |
| def vendor_id(self): |
| return self.ece.vendor_id |
| @vendor_id.setter |
| def vendor_id(self, val): |
| self.ece.vendor_id = val |
| |
| @property |
| def options(self): |
| return self.ece.options |
| @options.setter |
| def options(self, val): |
| self.ece.options = val |
| |
| @property |
| def comp_mask(self): |
| return self.ece.comp_mask |
| @comp_mask.setter |
| def comp_mask(self, val): |
| self.ece.comp_mask = val |
| |
| def __str__(self): |
| print_format = '{:22}: 0x{:<20x}\n' |
| return 'ECE:\n' +\ |
| print_format.format('Vendor ID', self.ece.vendor_id) +\ |
| print_format.format('Options', self.ece.options) +\ |
| print_format.format('Comp Mask', self.ece.comp_mask) |
| |
| |
| cdef class QP(PyverbsCM): |
| def __init__(self, object creator not None, object init_attr not None, |
| QPAttr qp_attr=None): |
| """ |
| Initializes a QP object and performs state transitions according to |
| user request. |
| A C ibv_qp object will be created using the provided init_attr. |
| If a qp_attr object is provided, pyverbs will consider this a hint to |
| transit the QP's state as far as possible towards RTS: |
| - In case of UD and Raw Packet QP types, if a qp_attr is provided the |
| QP will be returned in RTS state. |
| - In case of connected QPs (RC, UC), remote QPN is needed for INIT2RTR |
| transition, so if a qp_attr is provided, the QP will be returned in |
| INIT state. |
| :param creator: The object creating the QP. Can be of type PD so |
| ibv_create_qp will be used or of type Context, so |
| ibv_create_qp_ex will be used. |
| :param init_attr: QP initial attributes of type QPInitAttr (when |
| created using PD) or QPInitAttrEx (when created |
| using Context). |
| :param qp_attr: Optional QPAttr object. Will be used for QP state |
| transitions after creation. |
| :return: An initialized QP object |
| """ |
| cdef PD pd |
| cdef Context ctx |
| super().__init__() |
| self.mws = weakref.WeakSet() |
| self.flows = weakref.WeakSet() |
| self.dr_actions = weakref.WeakSet() |
| self.update_cqs(init_attr) |
| # QP initialization was not done by the provider, we should do it here |
| if self.qp == NULL: |
| # In order to use cdef'd methods, a proper casting must be done, |
| # let's infer the type. |
| if issubclass(type(creator), Context): |
| self._create_qp_ex(creator, init_attr) |
| if self.qp == NULL: |
| raise PyverbsRDMAErrno('Failed to create QP') |
| ctx = <Context>creator |
| self.context = ctx |
| ctx.add_ref(self) |
| if init_attr.pd is not None: |
| pd = <PD>init_attr.pd |
| pd.add_ref(self) |
| self.pd = pd |
| if init_attr.xrcd is not None: |
| xrcd = <XRCD>init_attr.xrcd |
| xrcd.add_ref(self) |
| self.xrcd = xrcd |
| |
| else: |
| self._create_qp(creator, init_attr) |
| if self.qp == NULL: |
| raise PyverbsRDMAErrno('Failed to create QP') |
| pd = <PD>creator |
| self.pd = pd |
| pd.add_ref(self) |
| self.context = None |
| if init_attr.srq is not None: |
| srq = <SRQ>init_attr.srq |
| srq.add_ref(self) |
| self.srq = srq |
| |
| if qp_attr is not None: |
| funcs = {e.IBV_QPT_RC: self.to_init, e.IBV_QPT_UC: self.to_init, |
| e.IBV_QPT_UD: self.to_rts, |
| e.IBV_QPT_XRC_RECV: self.to_init, |
| e.IBV_QPT_XRC_SEND: self.to_init, |
| e.IBV_QPT_RAW_PACKET: self.to_rts} |
| funcs[self.qp.qp_type](qp_attr) |
| |
| cdef update_cqs(self, init_attr): |
| cdef CQ cq |
| cdef CQEX cqex |
| if init_attr.send_cq is not None: |
| if type(init_attr.send_cq) == CQ: |
| cq = <CQ>init_attr.send_cq |
| cq.add_ref(self) |
| self.scq = cq |
| else: |
| cqex = <CQEX>init_attr.send_cq |
| cqex.add_ref(self) |
| self.scq = cqex |
| if init_attr.send_cq != init_attr.recv_cq and init_attr.recv_cq is not None: |
| if type(init_attr.recv_cq) == CQ: |
| cq = <CQ>init_attr.recv_cq |
| cq.add_ref(self) |
| self.rcq = cq |
| else: |
| cqex = <CQEX>init_attr.recv_cq |
| cqex.add_ref(self) |
| self.rcq = cqex |
| |
| def _create_qp(self, PD pd, QPInitAttr attr): |
| self.qp = v.ibv_create_qp(pd.pd, &attr.attr) |
| |
| def _create_qp_ex(self, Context ctx, QPInitAttrEx attr): |
| self.qp = v.ibv_create_qp_ex(ctx.context, &attr.attr) |
| |
| cdef add_ref(self, obj): |
| if isinstance(obj, MW): |
| self.mws.add(obj) |
| elif isinstance(obj, Flow): |
| self.flows.add(obj) |
| else: |
| raise PyverbsError('Unrecognized object type') |
| |
| def __dealloc__(self): |
| self.close() |
| |
| cpdef close(self): |
| if self.qp != NULL: |
| if self.logger: |
| self.logger.debug('Closing QP') |
| close_weakrefs([self.mws, self.flows, self.dr_actions]) |
| rc = v.ibv_destroy_qp(self.qp) |
| if rc: |
| raise PyverbsRDMAError('Failed to destroy QP', rc) |
| self.qp = NULL |
| self.pd = None |
| self.context = None |
| self.scq = None |
| self.rcq = None |
| |
| def _get_comp_mask(self, dst): |
| masks = {e.IBV_QPT_RC: {'INIT': e.IBV_QP_PKEY_INDEX | e.IBV_QP_PORT |\ |
| e.IBV_QP_ACCESS_FLAGS, 'RTR': e.IBV_QP_AV |\ |
| e.IBV_QP_PATH_MTU | e.IBV_QP_DEST_QPN |\ |
| e.IBV_QP_RQ_PSN |\ |
| e.IBV_QP_MAX_DEST_RD_ATOMIC |\ |
| e.IBV_QP_MIN_RNR_TIMER, |
| 'RTS': e.IBV_QP_TIMEOUT |\ |
| e.IBV_QP_RETRY_CNT | e.IBV_QP_RNR_RETRY |\ |
| e.IBV_QP_SQ_PSN | e.IBV_QP_MAX_QP_RD_ATOMIC}, |
| e.IBV_QPT_UC: {'INIT': e.IBV_QP_PKEY_INDEX | e.IBV_QP_PORT |\ |
| e.IBV_QP_ACCESS_FLAGS, 'RTR': e.IBV_QP_AV |\ |
| e.IBV_QP_PATH_MTU | e.IBV_QP_DEST_QPN |\ |
| e.IBV_QP_RQ_PSN, 'RTS': e.IBV_QP_SQ_PSN}, |
| e.IBV_QPT_UD: {'INIT': e.IBV_QP_PKEY_INDEX | e.IBV_QP_PORT |\ |
| e.IBV_QP_QKEY, 'RTR': 0, |
| 'RTS': e.IBV_QP_SQ_PSN}, |
| e.IBV_QPT_RAW_PACKET: {'INIT': e.IBV_QP_PORT, 'RTR': 0, |
| 'RTS': 0}, |
| e.IBV_QPT_XRC_RECV: {'INIT': e.IBV_QP_PKEY_INDEX |\ |
| e.IBV_QP_PORT | e.IBV_QP_ACCESS_FLAGS, |
| 'RTR': e.IBV_QP_AV | e.IBV_QP_PATH_MTU |\ |
| e.IBV_QP_DEST_QPN | e.IBV_QP_RQ_PSN | \ |
| e.IBV_QP_MAX_DEST_RD_ATOMIC |\ |
| e.IBV_QP_MIN_RNR_TIMER, |
| 'RTS': e.IBV_QP_TIMEOUT | e.IBV_QP_SQ_PSN }, |
| e.IBV_QPT_XRC_SEND: {'INIT': e.IBV_QP_PKEY_INDEX |\ |
| e.IBV_QP_PORT | e.IBV_QP_ACCESS_FLAGS, |
| 'RTR': e.IBV_QP_AV | e.IBV_QP_PATH_MTU |\ |
| e.IBV_QP_DEST_QPN | e.IBV_QP_RQ_PSN, |
| 'RTS': e.IBV_QP_TIMEOUT |\ |
| e.IBV_QP_RETRY_CNT | e.IBV_QP_RNR_RETRY |\ |
| e.IBV_QP_SQ_PSN | e.IBV_QP_MAX_QP_RD_ATOMIC}} |
| |
| return masks[self.qp.qp_type][dst] | e.IBV_QP_STATE |
| |
| def to_init(self, QPAttr qp_attr): |
| """ |
| Modify the current QP's state to INIT. If the current state doesn't |
| support transition to INIT, an exception will be raised. |
| The comp mask provided to the kernel includes the needed bits for 2INIT |
| transition for this QP type. |
| :param qp_attr: QPAttr object containing the needed attributes for |
| 2INIT transition |
| :return: None |
| """ |
| mask = self._get_comp_mask('INIT') |
| qp_attr.qp_state = e.IBV_QPS_INIT |
| rc = v.ibv_modify_qp(self.qp, &qp_attr.attr, mask) |
| if rc != 0: |
| raise PyverbsRDMAError('Failed to modify QP state to init', rc) |
| |
| def to_rtr(self, QPAttr qp_attr): |
| """ |
| Modify the current QP's state to RTR. It assumes that its current |
| state is INIT or RESET, in which case it will attempt a transition to |
| INIT prior to transition to RTR. As a result, if current state doesn't |
| support transition to INIT, an exception will be raised. |
| The comp mask provided to the kernel includes the needed bits for 2RTR |
| transition for this QP type. |
| :param qp_attr: QPAttr object containing the needed attributes for |
| 2RTR transition. |
| :return: None |
| """ |
| if self.qp_state != e.IBV_QPS_INIT: #assume reset |
| self.to_init(qp_attr) |
| mask = self._get_comp_mask('RTR') |
| qp_attr.qp_state = e.IBV_QPS_RTR |
| rc = v.ibv_modify_qp(self.qp, &qp_attr.attr, mask) |
| if rc != 0: |
| raise PyverbsRDMAError('Failed to modify QP state to RTR', rc) |
| |
| def to_rts(self, QPAttr qp_attr): |
| """ |
| Modify the current QP's state to RTS. It assumes that its current |
| state is either RTR, INIT or RESET. If current state is not RTR, to_rtr() |
| will be called. |
| The comp mask provided to the kernel includes the needed bits for 2RTS |
| transition for this QP type. |
| :param qp_attr: QPAttr object containing the needed attributes for |
| 2RTS transition. |
| :return: None |
| """ |
| if self.qp_state != e.IBV_QPS_RTR: #assume reset/init |
| self.to_rtr(qp_attr) |
| mask = self._get_comp_mask('RTS') |
| qp_attr.qp_state = e.IBV_QPS_RTS |
| rc = v.ibv_modify_qp(self.qp, &qp_attr.attr, mask) |
| if rc != 0: |
| raise PyverbsRDMAError('Failed to modify QP state to RTS', rc) |
| |
| def query(self, attr_mask): |
| """ |
| Query the QP |
| :param attr_mask: The minimum list of attributes to retrieve. Some |
| devices may return additional attributes as well |
| (see enum ibv_qp_attr_mask) |
| :return: (QPAttr, QPInitAttr) tuple containing the QP requested |
| attributes |
| """ |
| attr = QPAttr() |
| init_attr = QPInitAttr() |
| rc = v.ibv_query_qp(self.qp, &attr.attr, attr_mask, &init_attr.attr) |
| if rc != 0: |
| raise PyverbsRDMAError('Failed to query QP', rc) |
| return attr, init_attr |
| |
| def modify(self, QPAttr qp_attr not None, comp_mask): |
| """ |
| Modify the QP |
| :param qp_attr: A QPAttr object with updated values to be applied to |
| the QP |
| :param comp_mask: A bitmask specifying which QP attributes should be |
| modified (see enum ibv_qp_attr_mask) |
| :return: None |
| """ |
| rc = v.ibv_modify_qp(self.qp, &qp_attr.attr, comp_mask) |
| if rc != 0: |
| raise PyverbsRDMAError('Failed to modify QP', rc) |
| |
| def post_recv(self, RecvWR wr not None, RecvWR bad_wr=None): |
| """ |
| Post a receive WR on the QP. |
| :param wr: The work request to post |
| :param bad_wr: A RecvWR object to hold the bad WR if it is available in |
| case of a failure |
| :return: None |
| """ |
| cdef v.ibv_recv_wr *my_bad_wr |
| # In order to provide a pointer to a pointer, use a temporary cdef'ed |
| # variable. |
| rc = v.ibv_post_recv(self.qp, &wr.recv_wr, &my_bad_wr) |
| if rc != 0: |
| if (bad_wr): |
| memcpy(&bad_wr.recv_wr, my_bad_wr, sizeof(bad_wr.recv_wr)) |
| raise PyverbsRDMAError('Failed to post recv', rc) |
| |
| def post_send(self, SendWR wr not None, SendWR bad_wr=None): |
| """ |
| Post a send WR on the QP. |
| :param wr: The work request to post |
| :param bad_wr: A SendWR object to hold the bad WR if it is available in |
| case of a failure |
| :return: None |
| """ |
| # In order to provide a pointer to a pointer, use a temporary cdef'ed |
| # variable. |
| cdef v.ibv_send_wr *my_bad_wr |
| rc = v.ibv_post_send(self.qp, &wr.send_wr, &my_bad_wr) |
| if rc != 0: |
| if (bad_wr): |
| memcpy(&bad_wr.send_wr, my_bad_wr, sizeof(bad_wr.send_wr)) |
| raise PyverbsRDMAError('Failed to post send', rc) |
| |
| def set_ece(self, ECE ece): |
| """ |
| Set ECE options and use them for QP configuration stage |
| :param ece: The requested ECE values. |
| :return: None |
| """ |
| if ece.ece.vendor_id == 0: |
| return |
| |
| rc = v.ibv_set_ece(self.qp, &ece.ece) |
| if rc != 0: |
| raise PyverbsRDMAError('Failed to set ECE', rc) |
| |
| def query_ece(self): |
| """ |
| Query QPs ECE options |
| :return: ECE object with this QP ece configuration. |
| """ |
| ece = ECE() |
| rc = v.ibv_query_ece(self.qp, &ece.ece) |
| if rc != 0: |
| raise PyverbsRDMAError('Failed to query ECE', rc) |
| return ece |
| |
| def bind_mw(self, MW mw not None, MWBind mw_bind): |
| """ |
| Bind Memory window type 1. |
| :param mw: The memory window to bind. |
| :param mw_bind: MWBind object, includes the bind attributes. |
| :return: None |
| """ |
| rc = v.ibv_bind_mw(self.qp, mw.mw, &mw_bind.mw_bind) |
| if rc != 0: |
| raise PyverbsRDMAError('Failed to Bind MW', rc) |
| |
| def query_data_in_order(self, op, flags=0): |
| """ |
| Query if QP data is guaranteed to be in order. |
| :param op: Operation type. |
| :param flags: Extra field for future input. For now must be 0. |
| :return: 1 in case the data is guaranteed to be in order, 0 otherwise. |
| """ |
| return v.ibv_query_qp_data_in_order(self.qp, op, flags) |
| |
| @property |
| def qp_type(self): |
| return self.qp.qp_type |
| |
| @property |
| def qp_state(self): |
| return self.qp.state |
| |
| @property |
| def qp_num(self): |
| return self.qp.qp_num |
| |
| def __str__(self): |
| print_format = '{:22}: {:<20}\n' |
| return print_format.format('QP type', qp_type_to_str(self.qp_type)) +\ |
| print_format.format(' number', self.qp_num) +\ |
| print_format.format(' state', qp_state_to_str(self.qp_state)) |
| |
| |
| cdef class DataBuffer(PyverbsCM): |
| def __init__(self, addr, length): |
| super().__init__() |
| self.data.addr = PyLong_AsVoidPtr(addr) |
| self.data.length = length |
| |
| |
| cdef class QPEx(QP): |
| def __init__(self, object creator not None, object init_attr not None, |
| QPAttr qp_attr=None): |
| """ |
| Initializes a QPEx object. Since this is an extension of a QP, QP |
| creation is done in the parent class. The extended QP is retrieved by |
| casting the ibv_qp to ibv_qp_ex. |
| :return: An initialized QPEx object |
| """ |
| super().__init__(creator, init_attr, qp_attr) |
| if init_attr.ind_table is not None: |
| ind_table = <RwqIndTable>init_attr.ind_table |
| ind_table.add_ref(self) |
| self.ind_table = ind_table |
| if init_attr.comp_mask & v.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS: |
| self.qp_ex = v.ibv_qp_to_qp_ex(self.qp) |
| if self.qp_ex == NULL: |
| raise PyverbsRDMAErrno('Failed to create extended QP') |
| else: |
| self.logger.debug('qp_ex is not accessible since IBV_QP_INIT_ATTR_SEND_OPS_FLAGS was not passed.') |
| |
| @property |
| def comp_mask(self): |
| return self.qp_ex.comp_mask |
| @comp_mask.setter |
| def comp_mask(self, val): |
| self.qp_ex.comp_mask = val |
| |
| @property |
| def wr_id(self): |
| return self.qp_ex.wr_id |
| @wr_id.setter |
| def wr_id(self, val): |
| self.qp_ex.wr_id = val |
| |
| @property |
| def wr_flags(self): |
| return self.qp_ex.wr_flags |
| @wr_flags.setter |
| def wr_flags(self, val): |
| self.qp_ex.wr_flags = val |
| |
| @property |
| def ind_table(self): |
| return self.ind_table |
| |
| def wr_atomic_cmp_swp(self, rkey, remote_addr, compare, swap): |
| v.ibv_wr_atomic_cmp_swp(self.qp_ex, rkey, remote_addr, compare, swap) |
| |
| def wr_atomic_fetch_add(self, rkey, remote_addr, add): |
| v.ibv_wr_atomic_fetch_add(self.qp_ex, rkey, remote_addr, add) |
| |
| def wr_bind_mw(self, MW mw, rkey, MWBindInfo bind_info): |
| cdef v.ibv_mw_bind_info *info |
| info = &bind_info.info |
| v.ibv_wr_bind_mw(self.qp_ex, <v.ibv_mw*>mw.mw, rkey, |
| <v.ibv_mw_bind_info*>info) |
| self.add_ref(mw) |
| |
| def wr_local_inv(self, invalidate_rkey): |
| v.ibv_wr_local_inv(self.qp_ex, invalidate_rkey) |
| |
| def wr_rdma_read(self, rkey, remote_addr): |
| v.ibv_wr_rdma_read(self.qp_ex, rkey, remote_addr) |
| |
| def wr_rdma_write(self, rkey, remote_addr): |
| v.ibv_wr_rdma_write(self.qp_ex, rkey, remote_addr) |
| |
| def wr_rdma_write_imm(self, rkey, remote_addr, data): |
| cdef unsigned int imm_data = htobe32(data) |
| v.ibv_wr_rdma_write_imm(self.qp_ex, rkey, remote_addr, imm_data) |
| |
| def wr_send(self): |
| v.ibv_wr_send(self.qp_ex) |
| |
| def wr_send_imm(self, data): |
| cdef unsigned int imm_data = htobe32(data) |
| return v.ibv_wr_send_imm(self.qp_ex, imm_data) |
| |
| def wr_send_inv(self, invalidate_rkey): |
| v.ibv_wr_send_inv(self.qp_ex, invalidate_rkey) |
| |
| def wr_send_tso(self, hdr, hdr_sz, mss): |
| ptr = PyLong_AsVoidPtr(hdr) |
| v.ibv_wr_send_tso(self.qp_ex, ptr, hdr_sz, mss) |
| |
| def wr_set_ud_addr(self, AH ah, remote_qpn, remote_rkey): |
| v.ibv_wr_set_ud_addr(self.qp_ex, ah.ah, remote_qpn, remote_rkey) |
| |
| def wr_set_xrc_srqn(self, remote_srqn): |
| v.ibv_wr_set_xrc_srqn(self.qp_ex, remote_srqn) |
| |
| def wr_set_inline_data(self, addr, length): |
| ptr = PyLong_AsVoidPtr(addr) |
| v.ibv_wr_set_inline_data(self.qp_ex, ptr, length) |
| |
| def wr_set_inline_data_list(self, num_buf, buf_list): |
| cdef v.ibv_data_buf *data = NULL |
| data = <v.ibv_data_buf*>malloc(num_buf * sizeof(v.ibv_data_buf)) |
| if data == NULL: |
| raise PyverbsError('Failed to allocate data buffer') |
| for i in range(num_buf): |
| data_buf = <DataBuffer>buf_list[i] |
| data[i].addr = data_buf.data.addr |
| data[i].length = data_buf.data.length |
| v.ibv_wr_set_inline_data_list(self.qp_ex, num_buf, data) |
| free(data) |
| |
| def wr_set_sge(self, SGE sge not None): |
| v.ibv_wr_set_sge(self.qp_ex, sge.lkey, sge.addr, sge.length) |
| |
| def wr_set_sge_list(self, num_sge, sg_list): |
| cdef v.ibv_sge *sge = NULL |
| sge = <v.ibv_sge*>malloc(num_sge * sizeof(v.ibv_sge)) |
| if sge == NULL: |
| raise PyverbsError('Failed to allocate SGE buffer') |
| for i in range(num_sge): |
| sge[i].addr = sg_list[i].addr |
| sge[i].length = sg_list[i].length |
| sge[i].lkey = sg_list[i].lkey |
| v.ibv_wr_set_sge_list(self.qp_ex, num_sge, sge) |
| free(sge) |
| |
| def wr_start(self): |
| v.ibv_wr_start(self.qp_ex) |
| |
| def wr_complete(self): |
| rc = v.ibv_wr_complete(self.qp_ex) |
| if rc != 0: |
| raise PyverbsRDMAError('ibv_wr_complete failed.', rc) |
| |
| def wr_abort(self): |
| v.ibv_wr_abort(self.qp_ex) |
| |
| |
| def _copy_caps(QPCap src, dst): |
| """ |
| Copy the QPCaps values of src into the inner ibv_qp_cap struct of dst. |
| Since both ibv_qp_init_attr and ibv_qp_attr have an inner ibv_qp_cap inner |
| struct, they can both be used. |
| :param src: A QPCap object |
| :param dst: A QPInitAttr / QPInitAttrEx / QPAttr object |
| :return: None |
| """ |
| # we're assigning to C structs here, we must have type-specific objects in |
| # order to do that. Instead of having this function smaller but in 3 |
| # classes, it appears here once. |
| cdef QPInitAttr qia |
| cdef QPInitAttrEx qiae |
| cdef QPAttr qa |
| if src is None: |
| return |
| if type(dst) == QPInitAttr: |
| qia = <QPInitAttr>dst |
| qia.attr.cap.max_send_wr = src.cap.max_send_wr |
| qia.attr.cap.max_recv_wr = src.cap.max_recv_wr |
| qia.attr.cap.max_send_sge = src.cap.max_send_sge |
| qia.attr.cap.max_recv_sge = src.cap.max_recv_sge |
| qia.attr.cap.max_inline_data = src.cap.max_inline_data |
| elif type(dst) == QPInitAttrEx: |
| qiae = <QPInitAttrEx>dst |
| qiae.attr.cap.max_send_wr = src.cap.max_send_wr |
| qiae.attr.cap.max_recv_wr = src.cap.max_recv_wr |
| qiae.attr.cap.max_send_sge = src.cap.max_send_sge |
| qiae.attr.cap.max_recv_sge = src.cap.max_recv_sge |
| qiae.attr.cap.max_inline_data = src.cap.max_inline_data |
| else: |
| qa = <QPAttr>dst |
| qa.attr.cap.max_send_wr = src.cap.max_send_wr |
| qa.attr.cap.max_recv_wr = src.cap.max_recv_wr |
| qa.attr.cap.max_send_sge = src.cap.max_send_sge |
| qa.attr.cap.max_recv_sge = src.cap.max_recv_sge |
| qa.attr.cap.max_inline_data = src.cap.max_inline_data |
