from libc.stdint cimport uintptr_t, uint8_t
from libc.string cimport memset
import weakref

import ctypes
from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError, PyverbsRDMAError
from pyverbs.qp cimport QPInitAttr, QPAttr, ECE
from pyverbs.base import PyverbsRDMAErrno
from pyverbs.base cimport close_weakrefs
cimport pyverbs.libibverbs_enums as e
cimport pyverbs.librdmacm_enums as ce
from pyverbs.addr cimport AH, AHAttr
from pyverbs.device cimport Context
cimport pyverbs.libibverbs as v
cimport pyverbs.librdmacm as cm
from pyverbs.pd cimport PD
from pyverbs.mr cimport MR
from pyverbs.cq cimport WC


cdef class ConnParam(PyverbsObject):

    def __init__(self, resources=1, depth=1, flow_control=0, retry=5,
                 rnr_retry=5, srq=0, qp_num=0, data_len=0):
        """
        Initialize a ConnParam object over an underlying rdma_conn_param
        C object which contains connection parameters. There are a few types of
        port spaces in RDMACM: RDMA_PS_TCP, RDMA_PS_UDP, RDMA_PS_IB and
        RDMA_PS_IPOIB. RDMA_PS_TCP resembles RC QP connection, which provides
        reliable, connection-oriented QP communication. This object applies only
        to RDMA_PS_TCP port space.
        :param resources: Max outstanding RDMA read and atomic ops that local
                          side will accept from the remote side.
        :param depth: Max outstanding RDMA read and atomic ops that local side
                      will have to the remote side.
        :param flow_control: Specifies if hardware flow control is available.
        :param retry: Max number of times that a send, RDMA or atomic op from
                      the remote peer should be retried.
        :param rnr_retry: The maximum number of times that a send operation from
                          the remote peer should be retried on a connection
                          after receiving a receiver not ready (RNR) error.
        :param srq: Specifies if the QP using shared receive queue, ignored if
                    the QP created by CMID.
        :param qp_num: Specifies the QP number, ignored if the QP created by
                       CMID.
        :param data_len: Specifies the private data length.
                         RDMA_PS_TCP connect: 56
                                     accept:  196
                         RDMA_PS_UDP connect: 180
                                     accept:  136
        :return: ConnParam object
        """
        super().__init__()
        memset(&self.conn_param, 0, sizeof(cm.rdma_conn_param))
        self.conn_param.responder_resources = resources
        self.conn_param.initiator_depth = depth
        self.conn_param.flow_control = flow_control
        self.conn_param.retry_count = retry
        self.conn_param.rnr_retry_count = rnr_retry
        self.conn_param.srq = srq
        self.conn_param.qp_num = qp_num
        self.data = (ctypes.c_char * data_len)()

    @property
    def qpn(self):
        return self.conn_param.qp_num
    @qpn.setter
    def qpn(self, val):
        self.conn_param.qp_num = val

    @property
    def private_data(self):
        data_array = bytearray(self.conn_param.private_data_len)
        cdef const unsigned char *p = <const unsigned char*>self.conn_param.private_data
        for i in range(self.conn_param.private_data_len):
            data_array[i] = p[i]
        return bytes(data_array)

    def set_private_data(self, data):
        if (min(len(self.data), len(data)) == 0):
            return

        for i in range(len(self.data)):
            self.data[i] = 0

        for i in range(min(len(self.data), len(data))):
            self.data[i] = data[i]

        cdef size_t ptr = ctypes.addressof(self.data)
        self.conn_param.private_data = <const void*>ptr
        self.conn_param.private_data_len = min(len(self.data), len(data))

    def __str__(self):
        print_format  = '{:<4}: {:<4}\n'
        return '{}: {}\n'.format('Connection parameters', "") +\
               print_format.format('responder resources', self.conn_param.responder_resources) +\
               print_format.format('initiator depth', self.conn_param.initiator_depth) +\
               print_format.format('flow control', self.conn_param.flow_control) +\
               print_format.format('retry count', self.conn_param.retry_count) +\
               print_format.format('rnr retry count', self.conn_param.rnr_retry_count) +\
               print_format.format('srq', self.conn_param.srq) +\
               print_format.format('qp number', self.conn_param.qp_num)


cdef class JoinMCAttrEx(PyverbsObject):

    def __init__(self, AddrInfo addr not None, comp_mask=0, join_flags=0):
        """
        Initialize a JoinMCAttrEx object over an underlying
        rdma_cm_join_mc_attr_ex C object which contains the extended join
        multicast attributes.
        :param addr: Multicast address identifying the group to join.
        :param comp_mask: Bitwise OR between "rdma_cm_join_mc_attr_mask" enum.
        :param join_flags: Single flag from "rdma_cm_mc_join_flags" enum.
                           Indicates the type of the join requests.
        """
        super().__init__()
        self.join_mc_attr_ex.addr = addr.addr_info.ai_src_addr
        self.join_mc_attr_ex.comp_mask = comp_mask
        self.join_mc_attr_ex.join_flags = join_flags

    @property
    def join_flags(self):
        return self.join_mc_attr_ex.join_flags
    @join_flags.setter
    def join_flags(self, val):
        self.join_mc_attr_ex.join_flags = val

    @property
    def comp_mask(self):
        return self.join_mc_attr_ex.comp_mask
    @comp_mask.setter
    def comp_mask(self, val):
        self.join_mc_attr_ex.comp_mask = val


cdef class UDParam(PyverbsObject):

    def __init__(self, CMEvent cm_event not None):
        """
        Initialize a UDParam object over an underlying rdma_ud_param
        C object which contains UD connection parameters.
        :param cm_event: The creator of UDParam. When the active side gets
                         connection establishment event, the event contains
                         UDParam for the passive CMID details.
        :return: UDParam object
        """
        super().__init__()
        memset(&self.ud_param, 0, sizeof(cm.rdma_ud_param))
        self.ud_param = (<CMEvent>cm_event).event.param.ud

    @property
    def qp_num(self):
        return self.ud_param.qp_num

    @property
    def qkey(self):
        return self.ud_param.qkey

    @property
    def ah_attr(self):
        ah_attr = AHAttr()
        ah_attr.init_from_ud_param(self)
        return ah_attr


cdef class AddrInfo(PyverbsObject):

    def __init__(self, src=None, dst=None, src_service=None, dst_service=None,
                 port_space=0, flags=0):
        """
        Initialize an AddrInfo object over an underlying rdma_addrinfo C object.
        :param src: Name, dotted-decimal IPv4 or IPv6 hex address to bind to.
        :param dst: Name, dotted-decimal IPv4 or IPv6 hex address to connect to.
        :param src_service: The service name or port number of the source
                            address.
        :param dst_service: The service name or port number of the destination
                            address.
        :param port_space: RDMA port space used (RDMA_PS_UDP or RDMA_PS_TCP).
        :param flags: Hint flags which control the operation.
        :return: An AddrInfo object which contains information needed to
        establish communication.
        """
        cdef char* src_srvc = NULL
        cdef char* dst_srvc = NULL
        cdef char* src_addr = NULL
        cdef char* dst_addr = NULL
        cdef cm.rdma_addrinfo hints
        cdef cm.rdma_addrinfo *hints_ptr = NULL
        cdef cm.rdma_addrinfo *res = NULL

        super().__init__()
        if src is not None:
            if isinstance(src, str):
                src = src.encode('utf-8')
            src_addr = <char*>src
        if dst is not None:
            if isinstance(dst, str):
                dst = dst.encode('utf-8')
            dst_addr = <char*>dst
        if src_service is not None:
            if isinstance(src_service, str):
                src_service = src_service.encode('utf-8')
            src_srvc = <char*>src_service
        if dst_service is not None:
            if isinstance(dst_service, str):
                dst_service = dst_service.encode('utf-8')
            dst_srvc = <char*>dst_service

        hints_ptr = &hints
        memset(hints_ptr, 0, sizeof(cm.rdma_addrinfo))
        hints.ai_port_space = port_space
        hints.ai_flags = flags
        if flags & ce.RAI_PASSIVE:
            ret = cm.rdma_getaddrinfo(src_addr, src_srvc, hints_ptr,
                                      &self.addr_info)
        else:
            if src:
                hints.ai_flags |= ce.RAI_PASSIVE
                ret = cm.rdma_getaddrinfo(src_addr, src_srvc, hints_ptr, &res)
                if ret != 0:
                    raise PyverbsRDMAErrno('Failed to get Address Info')
                hints.ai_src_addr = <cm.sockaddr*>res.ai_src_addr
                hints.ai_src_len = res.ai_src_len
                hints.ai_flags &= ~ce.RAI_PASSIVE
            ret = cm.rdma_getaddrinfo(dst_addr, dst_srvc, hints_ptr,
                                      &self.addr_info)
            if src:
                cm.rdma_freeaddrinfo(res)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to get Address Info')

    def __dealloc__(self):
        self.close()

    cpdef close(self):
        if self.addr_info != NULL:
            if self.logger:
                self.logger.debug('Closing AddrInfo')
            cm.rdma_freeaddrinfo(self.addr_info)
        self.addr_info = NULL


cdef class CMEvent(PyverbsObject):

    def __init__(self, CMEventChannel channel):
        """
        Initialize a CMEvent object over an underlying rdma_cm_event C object
        :param channel: Event Channel on which this event has been received
        :return: CMEvent object
        """
        super().__init__()
        ret = cm.rdma_get_cm_event(channel.event_channel, &self.event)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to create CMEvent')
        self.logger.debug('Created a CMEvent')

    def __dealloc__(self):
        self.close()

    cpdef close(self):
        if self.event != NULL:
            if self.logger:
                self.logger.debug('Closing CMEvent')
            self.ack_cm_event()
            self.event = NULL

    @property
    def event_type(self):
        return self.event.event

    def ack_cm_event(self):
        """
        Free a communication event. This call frees the event structure and any
        memory that it references.
        :return: None
        """
        ret = cm.rdma_ack_cm_event(self.event)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to Acknowledge Event - {}'
                                   .format(self.event_str()))
        self.event = NULL

    def event_str(self):
        if self.event == NULL:
            return ''
        return (<bytes>cm.rdma_event_str(self.event_type)).decode()

    @property
    def private_data(self):
        data_array = bytearray(self.event.param.conn.private_data_len)
        cdef const unsigned char *p = <const unsigned char*>self.event.param.conn.private_data
        for i in range(self.event.param.conn.private_data_len):
            data_array[i] = p[i]
        return bytes(data_array)


cdef class CMEventChannel(PyverbsObject):

    def __init__(self):
        """
        Initialize a CMEventChannel object over an underlying rdma_event_channel
        C object.
        :return: EventChannel object
        """
        super().__init__()
        self.event_channel = cm.rdma_create_event_channel()
        if self.event_channel == NULL:
            raise PyverbsRDMAErrno('Failed to create CMEventChannel')
        self.logger.debug('Created a CMEventChannel')

    def __dealloc__(self):
        self.close()

    cpdef close(self):
        if self.event_channel != NULL:
            if self.logger:
                self.logger.debug('Closing CMEventChannel')
            cm.rdma_destroy_event_channel(self.event_channel)
            self.event_channel = NULL


cdef class CMID(PyverbsCM):

    def __init__(self, object creator=None, QPInitAttr qp_init_attr=None,
                 PD pd=None, port_space=ce.RDMA_PS_TCP, CMID listen_id=None):
        """
        Initialize a CMID object over an underlying rdma_cm_id C object.
        This is the main RDMA CM object which provides most of the rdmacm API.
        Currently only synchronous RDMA_PS_TCP communication supported.
        Notes: User-specific context, currently not supported.
        :param creator: For synchronous communication we need AddrInfo object in
                        order to establish connection. We allow creator to be
                        None for inner usage, see get_request method.
        :param qp_init_attr: Optional initial QP attributes of CMID
                             associated QP.
        :param pd: Optional parameter, a PD to be associated with this CMID.
        :param port_space: RDMA port space.
        :param listen_id: When passive side establishes a connection, it creates
                          a new CMID. listen_id is used to initialize the new
                          CMID.
        :return: CMID object for synchronous communication.
        """
        cdef v.ibv_qp_init_attr *init
        cdef v.ibv_pd *in_pd = NULL

        super().__init__()
        self.pd = None
        self.ctx = None
        self.event_channel = None
        self.mrs = weakref.WeakSet()
        if creator is None:
            return
        elif isinstance(creator, AddrInfo):
            init = NULL if qp_init_attr is None else &qp_init_attr.attr
            if pd is not None:
                in_pd = pd.pd
                self.pd = pd
            ret = cm.rdma_create_ep(&self.id, (<AddrInfo>creator).addr_info,
                                    in_pd, init)
            if ret != 0:
                raise PyverbsRDMAErrno('Failed to create CM ID')
            if not (<AddrInfo>creator).addr_info.ai_flags & ce.RAI_PASSIVE:
                self.ctx = Context(cmid=self)
                if self.pd is None:
                    self.pd = PD(self)
        elif isinstance(creator, CMEventChannel):
             self.event_channel = <CMEventChannel>creator
             ret = cm.rdma_create_id((<CMEventChannel>creator).event_channel,
                                     &self.id, NULL, port_space)
             if ret != 0:
                raise PyverbsRDMAErrno('Failed to create CM ID')
        elif isinstance(creator, CMEvent):
            if listen_id is None:
                raise PyverbsUserError('listen ID not provided')
            self.id = (<CMEvent>creator).event.id
            self.event_channel = listen_id.event_channel
            self.ctx = listen_id.ctx
            self.pd = listen_id.pd
        else:
            raise PyverbsRDMAErrno('Cannot create CM ID from {obj}'
                                    .format(obj=type(creator)))

    cdef add_ref(self, obj):
        if isinstance(obj, MR):
            self.mrs.add(obj)
        else:
            raise PyverbsError('Unrecognized object type')

    @property
    def dev_name(self):
        return str(v.ibv_get_device_name(self.id.verbs.device).decode('utf-8'))

    @property
    def port_num(self):
        return self.id.port_num

    @property
    def event_channel(self):
        return self.event_channel

    @property
    def context(self):
        return self.ctx

    @property
    def pd(self):
        return self.pd

    @property
    def qpn(self):
        if self.id.qp:
            return self.id.qp.qp_num
        return None

    def __dealloc__(self):
        self.close()

    cpdef close(self):
        if self.id != NULL:
            if self.logger:
                self.logger.debug('Closing CMID')
            if self.event_channel is None:
                cm.rdma_destroy_ep(self.id)
            else:
                if self.id.qp != NULL:
                    cm.rdma_destroy_qp(self.id)
                ret = cm.rdma_destroy_id(self.id)
                if ret != 0:
                    raise PyverbsRDMAErrno('Failed to close CMID')
            if self.ctx:
                (<Context>self.ctx).context = NULL
            if self.pd:
                (<PD>self.pd).pd = NULL
            close_weakrefs([self.mrs])
            self.id = NULL

    def get_request(self):
        """
        Retrieves the next pending connection request event. The call may only
        be used on listening CMIDs operating synchronously. If the call is
        successful, a new CMID representing the connection request will be
        returned to the user. The new CMID will reference event information
        associated with the request until the user calls reject, accept, or
        close on the newly created identifier.
        :return: New CMID representing the connection request.
        """
        to_conn = CMID()
        ret = cm.rdma_get_request(self.id, &to_conn.id)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to get request, no connection established')
        self.ctx = Context(cmid=to_conn)
        self.pd = PD(to_conn)
        return to_conn

    def bind_addr(self, AddrInfo lai not None):
        """
        Associate a source address with a CMID. If binding to a specific local
        address, the CMID will also be bound to a local RDMA device.
        :param lai: Local address information
        :return: None
        """
        ret = cm.rdma_bind_addr(self.id, lai.addr_info.ai_src_addr)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to Bind ID')
        # After bind address, cm_id contains ibv_context.
        # Now we can create Context object.
        if self.ctx is None:
            self.ctx = Context(cmid=self)
        if self.pd is None:
            self.pd = PD(self)

    def resolve_addr(self, AddrInfo rai not None, timeout_ms=2000):
        """
        Resolve destination and optional source addresses from IP addresses to
        an RDMA address. If successful, the specified rdma_cm_id will be bound
        to a local device.
        :param rai: Remote address information.
        :param timeout_ms: Time to wait for resolution to complete [msec]
        :return: None
        """
        ret = cm.rdma_resolve_addr(self.id, rai.addr_info.ai_src_addr,
                                   rai.addr_info.ai_dst_addr, timeout_ms)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to Resolve Address')

    def join_multicast(self, AddrInfo addr=None, JoinMCAttrEx mc_attr=None,
                       context=0):
        """
        Joins a multicast group and attaches an associated QP to the group.
        :param addr: Multicast address identifying the group to join.
        :param mc_attr: JoinMCAttrEx object is requierd to use
                        rdma_join_multicast_ex. This object contains the join
                        flags and the AddrInfo to join.
        :param context: User-defined context associated with the join request.
        :return: None
        """
        cdef cm.rdma_cm_join_mc_attr_ex  *mc_join_attr = NULL
        if not addr and not mc_attr:
            raise PyverbsUserError('Join to multicast must have AddrInfo or JoinMCAttrEx arguments')
        if not mc_attr:
            ret = cm.rdma_join_multicast(self.id, addr.addr_info.ai_src_addr,
                                         <void*><uintptr_t>context)
        else:
            ret = cm.rdma_join_multicast_ex(self.id, &mc_attr.join_mc_attr_ex,
                                            <void*><uintptr_t>context)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to Join multicast')

    def leave_multicast(self, AddrInfo addr not None):
        """
        Leaves a multicast group and detaches an associated QP from the group.
        :param addr: AddrInfo object, represent the multicast address that
                     identifies the group to leave.
        :return: None
        """
        ret = cm.rdma_leave_multicast(self.id, addr.addr_info.ai_src_addr)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to leave multicast')

    def resolve_route(self, timeout_ms=2000):
        """
        Resolve an RDMA route to the destination address in order to establish
        a connection. The destination must already have been resolved by calling
        resolve_addr. Thus this function is called on the client side after
        resolve_addr but before calling connect.
        :param timeout_ms: Time to wait for resolution to complete
        :return: None
        """
        ret = cm.rdma_resolve_route(self.id, timeout_ms)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to Resolve Route')
        # After resolve route, cm_id contains ibv_context.
        # Now we can create Context object.
        if self.ctx is None:
            self.ctx = Context(cmid=self)
        if self.pd is None:
            self.pd = PD(self)

    def listen(self, backlog=0):
        """
        Listen for incoming connection requests or datagram service lookup.
        The listen is restricted to the locally bound source address.
        :param backlog: The backlog of incoming connection requests
        :return: None
        """
        ret = cm.rdma_listen(self.id, backlog)
        if ret != 0:
            raise PyverbsRDMAErrno('Listen Failed')

    def connect(self, ConnParam param=None):
        """
        Initiates an active connection request to a remote destination.
        :param param: Optional connection parameters
        :return: None
        """
        cdef cm.rdma_conn_param *conn = &param.conn_param if param else NULL
        ret = cm.rdma_connect(self.id, conn)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to Connect')

    def disconnect(self):
        """
        Disconnects a connection and transitions any associated QP to error
        state.
        :return: None
        """
        ret = cm.rdma_disconnect(self.id)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to Disconnect')

    def accept(self, ConnParam param=None):
        """
        Is called from the listening side to accept a connection or datagram
        service lookup request.
        :param param: Optional connection parameters
        :return: None
        """
        cdef cm.rdma_conn_param *conn = &param.conn_param if param else NULL
        ret = cm.rdma_accept(self.id, conn)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to Accept Connection')

    def establish(self):
        """
        Complete an active connection request.
        If a QP has not been created on the CMID, this method should be
        called by the active side to complete the connection, after getting
        connect response event. This will trigger a connection established
        event on the passive side.
        This method should not be used on a CMID on which a QP has been
        created.
        """
        ret = cm.rdma_establish(self.id)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to Complete an active connection request')

    def set_local_ece(self, ECE ece):
        """
        Set local ECE paraemters to be used for REQ/REP communication.
        :param ece: ECE object with the requested configuration
        :return: None
        """
        rc = cm.rdma_set_local_ece(self.id, &ece.ece)
        if rc != 0:
            raise PyverbsRDMAErrno('Failed to set local ECE')

    def get_remote_ece(self):
        """
        Get ECE parameters as were received from the communication peer.
        :return: ECE object with the ece configuration
        """
        ece = ECE()
        rc = cm.rdma_get_remote_ece(self.id, &ece.ece)
        if rc != 0:
            raise PyverbsRDMAErrno('Failed to get remote ECE')
        return ece

    def create_qp(self, QPInitAttr qp_init not None):
        """
        Create a QP, which is associated with CMID.
        If CMID and qp_init don't hold any CQs, new CQs will be created and
        associated with CMID.
        If only qp_init provides CQs, they will not be associated with CMID.
        If both provide CQs they have to be using the same CQs.
        :param qp_init: QP init attributes
        """
        ret = cm.rdma_create_qp(self.id, (<PD>self.pd).pd, &qp_init.attr)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to Create QP')

    def query_qp(self, attr_mask):
        """
        Query QP using ibv_query_qp.
        :param attr_mask: Which attributes to query (use <enum name> enum)
        :return: A (QPAttr, QPInitAttr) tuple, containing the relevant QP info
        """
        attr = QPAttr()
        init_attr = QPInitAttr()
        rc = v.ibv_query_qp(self.id.qp, &attr.attr, attr_mask, &init_attr.attr)
        if rc != 0:
            raise PyverbsRDMAError('Failed to query QP', rc)
        return attr, init_attr

    def init_qp_attr(self, qp_state):
        """
        Initialize a QPAttr object used for state transitions of an external
        QP (a QP which was not created using CMID).
        When connecting external QPs using CMIDs both sides must call this
        method before QP state transition to RTR/RTS in order to obtain
        relevant QP attributes from CMID.
        :param qp_state: The QP's destination state
        :return: A (QPAttr, attr_mask) tuple, where attr_mask defines which
                 attributes of QPAttr are valid
        """
        cdef int attr_mask
        qp_attr = QPAttr()
        qp_attr.qp_state = qp_state

        rc = cm.rdma_init_qp_attr(self.id, &qp_attr.attr, &attr_mask)
        if rc != 0:
            raise PyverbsRDMAErrno('Failed to get QP attributes')
        return qp_attr, attr_mask

    def reg_msgs(self, size):
        """
        Registers a memory region for sending or receiving messages or for
        RDMA operations. The registered memory may then be posted to an CMID
        using post_send or post_recv methods.
        :param size: The total length of the memory to register
        :return: registered MR
        """
        return MR(self, size, e.IBV_ACCESS_LOCAL_WRITE)

    def reg_read(self, size=0):
        """
        Registers a memory region for sending or receiving messages or for
        remote read operations.
        :param size: The total length of the memory to register
        :return: registered MR
        """
        return MR(self, size, e.IBV_ACCESS_REMOTE_READ)

    def reg_write(self, size=0):
        """
        Registers a memory region for sending or receiving messages or for
        remote write operations.
        :param size: The total length of the memory to register
        :return: registered MR
        """
        return MR(self, size, e.IBV_ACCESS_REMOTE_WRITE)

    def post_recv(self, MR mr not None, length=None):
        """
        Posts a recv_wr via QP associated with CMID.
        Context param of rdma_post_recv C function currently not supported.
        :param mr: A valid MR object.
        :param length: length of buffer to recv (default: mr length).
        :return: None
        """
        if not length:
            length = mr.mr.length
        ret = cm.rdma_post_recv(self.id, NULL, mr.buf, length, mr.mr)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to Post Receive')

    def post_send(self, MR mr not None, flags=v.IBV_SEND_SIGNALED, length=None):
        """
        Posts a message via QP associated with CMID.
        Context param of rdma_post_send C function currently not supported.
        :param mr: A valid MR object which contains message to send.
        :param flags: flags for send work request.
        :param length: length of buffer to send (default: mr length).
        :return: None
        """
        if not length:
            length = mr.mr.length
        ret = cm.rdma_post_send(self.id, NULL, mr.buf, length, mr.mr,
                                flags)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to Post Send')

    def post_read(self, MR mr not None, length, remote_addr, rkey,
                  flags=0):
        """
        Post read WR using the CMIDs internal QP.
        :param mr: A valid MR object.
        :param length: length of buffer to send.
        :param remote_addr: The remote MR address.
        :param rkey: The remote MR rkey.
        :param flags: flags for send work request.
        :return: None
        """
        ret = cm.rdma_post_read(self.id, NULL, mr.buf, length, mr.mr,
                                flags, remote_addr, rkey)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to Post Read')

    def post_write(self, MR mr not None, length, remote_addr, rkey,
                   flags=0):
        """
        Post write WR using the CMIDs internal QP.
        :param mr: A valid MR object.
        :param length: length of buffer to send.
        :param remote_addr: The remote MR address.
        :param rkey: The remote MR rkey.
        :param flags: flags for send work request.
        :return: None
        """
        ret = cm.rdma_post_write(self.id, NULL, mr.buf, length, mr.mr,
                                 flags, remote_addr, rkey)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to Post Write')

    def post_ud_send(self, MR mr not None, AH ah not None, rqpn=0,
                     flags=v.IBV_SEND_SIGNALED, length=None):
        """
        Posts a message via UD QP associated with CMID to another UD QP.
        :param mr: A valid MR object which contains message to send.
        :param ah: The destination AH.
        :param rqpn: The remote QP number.
        :param flags: flags for send work request.
        :param length: length of buffer to send.
        :return: None
        """
        if not length:
            length = mr.mr.length
        ret = cm.rdma_post_ud_send(self.id, NULL, mr.buf, length, mr.mr,
                                   flags, ah.ah, rqpn)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to Post Send')

    def get_recv_comp(self):
        """
        Polls the receive CQ associated with CMID for a work completion.
        :return: The retrieved WC or None if there is no completions
        """
        cdef v.ibv_wc wc
        ret = cm.rdma_get_recv_comp(self.id, &wc)
        if ret < 0:
            raise PyverbsRDMAErrno('Failed to retrieve receive completion')
        elif ret == 0:
            return None
        return WC(wr_id=wc.wr_id, status=wc.status, opcode=wc.opcode,
                  vendor_err=wc.vendor_err, byte_len=wc.byte_len,
                  qp_num=wc.qp_num, src_qp=wc.src_qp,
                  imm_data=wc.imm_data, wc_flags=wc.wc_flags,
                  pkey_index=wc.pkey_index, slid=wc.slid, sl=wc.sl,
                  dlid_path_bits=wc.dlid_path_bits)

    def get_send_comp(self):
        """
        Polls the send CQ associated with CMID for a work completion.
        :return: The retrieved WC or None if there is no completions
        """
        cdef v.ibv_wc wc
        ret = cm.rdma_get_send_comp(self.id, &wc)
        if ret < 0:
            raise PyverbsRDMAErrno('Failed to retrieve send completion')
        elif ret == 0:
            return None
        return WC(wr_id=wc.wr_id, status=wc.status, opcode=wc.opcode,
                  vendor_err=wc.vendor_err, byte_len=wc.byte_len,
                  qp_num=wc.qp_num, src_qp=wc.src_qp,
                  imm_data=wc.imm_data, wc_flags=wc.wc_flags,
                  pkey_index=wc.pkey_index, slid=wc.slid, sl=wc.sl,
                  dlid_path_bits=wc.dlid_path_bits)

    def set_option(self, level, optname, optval, optlen):
        """
        Set communication options for a CMID.
        :param level: The protocol level of the option to set.
        :param optname: The name of the option to set.
        :param optval: The option data.
        :param optlen: The size of the data.
        """
        if optname != ce.RDMA_OPTION_ID_ACK_TIMEOUT:
            raise PyverbsUserError('Currently only RDMA_OPTION_ID_ACK_TIMEOUT is supported in Pyverbs.')
        cdef uint8_t value = optval
        ret = cm.rdma_set_option(self.id, level, optname, <void*>&value, optlen)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to set option')

    def reject(self, private_data=None):
        """
        Reject a connection or datagram service lookup request.
        :param private_data: Optional private data to send with the reject message.
        :param private_data_len: Size (in bytes) of the private data being sent.
        """
        data_len = len(private_data) if private_data else 0
        buffer = ctypes.create_string_buffer(data_len)
        if (private_data):
            buffer.value = private_data
        cdef size_t data_ptr = ctypes.addressof(buffer)

        ret = cm.rdma_reject(self.id, <const void*>data_ptr, data_len)
        if ret != 0:
            raise PyverbsRDMAErrno('Failed to Reject Connection')